Author: jianhe Date: Wed Jun 18 00:51:39 2014 New Revision: 1603338 URL: http://svn.apache.org/r1603338 Log: MAPREDUCE-5924. Changed TaskAttemptImpl to ignore TA_COMMIT_PENDING event at COMMIT_PENDING state. Contributed by Zhijie Shen
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1603338&r1=1603337&r2=1603338&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Jun 18 00:51:39 2014 @@ -264,6 +264,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5920. Add Xattr option in DistCp docs. (Yi Liu via cnauroth) + MAPREDUCE-5924. Changed TaskAttemptImpl to ignore TA_COMMIT_PENDING event + at COMMIT_PENDING state. (Zhijie Shen via jianhe) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1603338&r1=1603337&r2=1603338&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Jun 18 00:51:39 2014 @@ -335,6 +335,15 @@ public abstract class TaskAttemptImpl im .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION) + // AM is likely to receive duplicate TA_COMMIT_PENDINGs as the task attempt + // will re-send the commit message until it doesn't encounter any + // IOException and succeeds in delivering the commit message. + // Ignoring the duplicate commit message is a short-term fix. In long term, + // we need to make use of retry cache to help this and other MR protocol + // APIs that can be considered as @AtMostOnce. + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptEventType.TA_COMMIT_PENDING) // Transitions from SUCCESS_CONTAINER_CLEANUP state // kill and cleanup the container Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java?rev=1603338&r1=1603337&r2=1603338&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java Wed Jun 18 00:51:39 2014 @@ -112,6 +112,15 @@ public class TestMRApp { //wait for first attempt to commit pending app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING); + //re-send the commit pending signal to the task + app.getContext().getEventHandler().handle( + new TaskAttemptEvent( + attempt.getID(), + TaskAttemptEventType.TA_COMMIT_PENDING)); + + //the task attempt should be still at COMMIT_PENDING + app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING); + //send the done signal to the task app.getContext().getEventHandler().handle( new TaskAttemptEvent(