Author: bobby Date: Tue Apr 17 22:13:52 2012 New Revision: 1327306 URL: http://svn.apache.org/viewvc?rev=1327306&view=rev Log: svn merge -c 1327304 from trunk. FIXES: MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they are not being used any more (Ravi Prakash via bobby)
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Apr 17 22:13:52 2012 @@ -164,6 +164,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4151. RM scheduler web page should filter apps to those that are relevant to scheduling (Jason Lowe via tgraves) + MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they + are not being used any more (Ravi Prakash via bobby) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Tue Apr 17 22:13:52 2012 @@ -187,9 +187,6 @@ public class JobConf extends Configurati * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_TASK_ULIMIT} can be used to - * control the maximum virtual memory of the child processes. - * * The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass * other environment variables to the child processes. * @@ -210,9 +207,6 @@ public class JobConf extends Configurati * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_MAP_TASK_ULIMIT} can be used to - * control the maximum virtual memory of the map processes. - * * The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass * other environment variables to the map processes. */ @@ -230,9 +224,6 @@ public class JobConf extends Configurati * /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: * -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc * - * The configuration variable {@link #MAPRED_REDUCE_TASK_ULIMIT} can be used - * to control the maximum virtual memory of the reduce processes. - * * The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to * pass process environment variables to the reduce processes. */ @@ -242,36 +233,32 @@ public class JobConf extends Configurati public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m"; /** - * Configuration key to set the maximum virutal memory available to the child - * map and reduce tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_TASK_JAVA_OPTS}, else the VM might not start. - * - * @deprecated Use {@link #MAPRED_MAP_TASK_ULIMIT} or - * {@link #MAPRED_REDUCE_TASK_ULIMIT} + * @deprecated + * Configuration key to set the maximum virtual memory available to the child + * map and reduce tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ @Deprecated public static final String MAPRED_TASK_ULIMIT = "mapred.child.ulimit"; /** - * Configuration key to set the maximum virutal memory available to the - * map tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_MAP_TASK_JAVA_OPTS}, else the VM might not start. + * @deprecated + * Configuration key to set the maximum virtual memory available to the + * map tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ - public static final String MAPRED_MAP_TASK_ULIMIT = JobContext.MAP_ULIMIT; + @Deprecated + public static final String MAPRED_MAP_TASK_ULIMIT = "mapreduce.map.ulimit"; /** - * Configuration key to set the maximum virutal memory available to the - * reduce tasks (in kilo-bytes). - * - * Note: This must be greater than or equal to the -Xmx passed to the JavaVM - * via {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}, else the VM might not start. + * @deprecated + * Configuration key to set the maximum virtual memory available to the + * reduce tasks (in kilo-bytes). This has been deprecated and will no + * longer have any effect. */ - public static final String MAPRED_REDUCE_TASK_ULIMIT = - JobContext.REDUCE_ULIMIT; + @Deprecated + public static final String MAPRED_REDUCE_TASK_ULIMIT = + "mapreduce.reduce.ulimit"; /** @@ -1966,6 +1953,15 @@ public class JobConf extends Configurati + " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY + " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY); } + if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT)); + } + if(get(JobConf.MAPRED_MAP_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_MAP_TASK_ULIMIT)); + } + if(get(JobConf.MAPRED_REDUCE_TASK_ULIMIT) != null ) { + LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT)); + } } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Apr 17 22:13:52 2012 @@ -183,8 +183,6 @@ public interface MRJobConfig { public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts"; - public static final String MAP_ULIMIT = "mapreduce.map.ulimit"; - public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts"; public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script"; @@ -243,8 +241,6 @@ public interface MRJobConfig { public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts"; - public static final String REDUCE_ULIMIT = "mapreduce.reduce.ulimit"; - public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir"; public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts"; Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java Tue Apr 17 22:13:52 2012 @@ -314,8 +314,6 @@ public class ConfigUtil { new String[] {MRJobConfig.MAP_ENV}); Configuration.addDeprecation("mapred.map.child.java.opts", new String[] {MRJobConfig.MAP_JAVA_OPTS}); - Configuration.addDeprecation("mapred.map.child.ulimit", - new String[] {MRJobConfig.MAP_ULIMIT}); Configuration.addDeprecation("mapred.map.max.attempts", new String[] {MRJobConfig.MAP_MAX_ATTEMPTS}); Configuration.addDeprecation("mapred.map.task.debug.script", @@ -362,8 +360,6 @@ public class ConfigUtil { new String[] {MRJobConfig.REDUCE_ENV}); Configuration.addDeprecation("mapred.reduce.child.java.opts", new String[] {MRJobConfig.REDUCE_JAVA_OPTS}); - Configuration.addDeprecation("mapred.reduce.child.ulimit", - new String[] {MRJobConfig.REDUCE_ULIMIT}); Configuration.addDeprecation("mapred.reduce.max.attempts", new String[] {MRJobConfig.REDUCE_MAX_ATTEMPTS}); Configuration.addDeprecation("mapred.reduce.parallel.copies", Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Tue Apr 17 22:13:52 2012 @@ -411,9 +411,6 @@ /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. - Usage of -Djava.library.path can cause programs to no longer function if hadoop native libraries are used. These values should instead be set as part of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and @@ -432,20 +429,6 @@ </property> <property> - <name>mapred.child.ulimit</name> - <value></value> - <description>The maximum virtual memory, in KB, of a process launched by the - Map-Reduce framework. This can be used to control both the Mapper/Reducer - tasks and applications using Hadoop Pipes, Hadoop Streaming etc. - By default it is left unspecified to let cluster admins control it via - limits.conf and other such relevant mechanisms. - - Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to - JavaVM, else the VM might not start. - </description> -</property> - -<property> <name>mapreduce.admin.user.env</name> <value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native</value> <description>Expert: Additional execution environment entries for Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml Tue Apr 17 22:13:52 2012 @@ -403,9 +403,6 @@ For example, to enable verbose gc logging to a file named for the taskid in /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - - The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. </description> </property> @@ -420,20 +417,6 @@ </property> <property> - <name>mapred.child.ulimit</name> - <value></value> - <description>The maximum virtual memory, in KB, of a process launched by the - Map-Reduce framework. This can be used to control both the Mapper/Reducer - tasks and applications using Hadoop Pipes, Hadoop Streaming etc. - By default it is left unspecified to let cluster admins control it via - limits.conf and other such relevant mechanisms. - - Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to - JavaVM, else the VM might not start. - </description> -</property> - -<property> <name>mapreduce.map.log.level</name> <value>INFO</value> <description>The logging level for the map task. The allowed levels are: Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java Tue Apr 17 22:13:52 2012 @@ -33,12 +33,6 @@ class MapTaskRunner extends TaskRunner { super.getChildJavaOpts(jobConf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)); } - - @Override - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT, - super.getChildUlimit(jobConf)); - } @Override public String getChildEnv(JobConf jobConf) { Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Tue Apr 17 22:13:52 2012 @@ -41,12 +41,6 @@ class ReduceTaskRunner extends TaskRunne super.getChildJavaOpts(jobConf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS)); } - - @Override - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT, - super.getChildUlimit(jobConf)); - } @Override public String getChildEnv(JobConf jobConf) { Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java Tue Apr 17 22:13:52 2012 @@ -110,20 +110,7 @@ abstract class TaskRunner extends Thread public String getChildJavaOpts(JobConf jobConf, String defaultValue) { return jobConf.get(JobConf.MAPRED_TASK_JAVA_OPTS, defaultValue); } - - /** - * Get the maximum virtual memory of the child map/reduce tasks. - * @param jobConf job configuration - * @return the maximum virtual memory of the child task or <code>-1</code> if - * none is specified - * @deprecated Use limits specific to the map or reduce tasks set via - * {@link JobConf#MAPRED_MAP_TASK_ULIMIT} or - * {@link JobConf#MAPRED_REDUCE_TASK_ULIMIT} - */ - @Deprecated - public int getChildUlimit(JobConf jobConf) { - return jobConf.getInt(JobConf.MAPRED_TASK_ULIMIT, -1); - } + /** * Get the environment variables for the child map/reduce tasks. @@ -188,8 +175,7 @@ abstract class TaskRunner extends Thread tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf); - // set memory limit using ulimit if feasible and necessary ... - List<String> setup = getVMSetupCmd(); + List<String> setup = new ArrayList<String>(); // Set up the redirection of the task's stdout and stderr streams File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask()); @@ -311,26 +297,6 @@ abstract class TaskRunner extends Thread } /** - * @return - */ - private List<String> getVMSetupCmd() { - - int ulimit = getChildUlimit(conf); - if (ulimit <= 0) { - return null; - } - List<String> setup = null; - String[] ulimitCmd = Shell.getUlimitMemoryCommand(ulimit); - if (ulimitCmd != null) { - setup = new ArrayList<String>(); - for (String arg : ulimitCmd) { - setup.add(arg); - } - } - return setup; - } - - /** * Parse the given string and return an array of individual java opts. Split * on whitespace and replace the special string "@taskid@" with the task ID * given.