Author: jeagles
Date: Fri Aug 31 21:31:23 2012
New Revision: 1379622
URL: http://svn.apache.org/viewvc?rev=1379622&view=rev
Log:
svn merge -c 1379617 from trunk FIXES: MAPREDUCE-4604. In mapred-default,
mapreduce.map.maxattempts & mapreduce.reduce.maxattempts defaults are set to 4
as well as mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379622&r1=1379621&r2=1379622&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 21:31:23 2012
@@ -737,6 +737,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
Evans via tgraves)
+ MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts &
+ mapreduce.reduce.maxattempts defaults are set to 4 as well as
+ mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1379622&r1=1379621&r2=1379622&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
Fri Aug 31 21:31:23 2012
@@ -1357,7 +1357,7 @@ public class JobConf extends Configurati
* @return the maximum no. of failures of a given job per tasktracker.
*/
public int getMaxTaskFailuresPerTracker() {
- return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 4);
+ return getInt(JobContext.MAX_TASK_FAILURES_PER_TRACKER, 3);
}
/**
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1379622&r1=1379621&r2=1379622&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
Fri Aug 31 21:31:23 2012
@@ -797,9 +797,12 @@
<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
- <value>4</value>
+ <value>3</value>
<description>The number of task-failures on a tasktracker of a given job
- after which new tasks of that job aren't assigned to it.
+ after which new tasks of that job aren't assigned to it. It
+ MUST be less than mapreduce.map.maxattempts and
+ mapreduce.reduce.maxattempts otherwise the failed task will
+ never be tried on a different node.
</description>
</property>
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java?rev=1379622&r1=1379621&r2=1379622&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
Fri Aug 31 21:31:23 2012
@@ -21,6 +21,7 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
public class TestJobConf {
@@ -185,4 +186,19 @@ public class TestJobConf {
}
+
+ /**
+ * Ensure that by default JobContext.MAX_TASK_FAILURES_PER_TRACKER is less
+ * JobContext.MAP_MAX_ATTEMPTS and JobContext.REDUCE_MAX_ATTEMPTS so that
+ * failed tasks will be retried on other nodes
+ */
+ @Test
+ public void testMaxTaskFailuresPerTracker() {
+ JobConf jobConf = new JobConf(true);
+ Assert.assertTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was
"
+ + "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS"
+ ,jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxMapAttempts() &&
+ jobConf.getMaxTaskFailuresPerTracker() < jobConf.getMaxReduceAttempts()
+ );
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1379622&r1=1379621&r2=1379622&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
Fri Aug 31 21:31:23 2012
@@ -701,7 +701,7 @@
<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
- <value>4</value>
+ <value>3</value>
<description>The number of task-failures on a tasktracker of a given job
after which new tasks of that job aren't assigned to it.
</description>
