Author: jianhe
Date: Wed Jun 18 00:51:39 2014
New Revision: 1603338
URL: http://svn.apache.org/r1603338
Log:
MAPREDUCE-5924. Changed TaskAttemptImpl to ignore TA_COMMIT_PENDING event at
COMMIT_PENDING state. Contributed by Zhijie Shen
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1603338&r1=1603337&r2=1603338&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Jun 18
00:51:39 2014
@@ -264,6 +264,9 @@ Release 2.5.0 - UNRELEASED
MAPREDUCE-5920. Add Xattr option in DistCp docs. (Yi Liu via cnauroth)
+ MAPREDUCE-5924. Changed TaskAttemptImpl to ignore TA_COMMIT_PENDING event
+ at COMMIT_PENDING state. (Zhijie Shen via jianhe)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1603338&r1=1603337&r2=1603338&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Jun 18 00:51:39 2014
@@ -335,6 +335,15 @@ public abstract class TaskAttemptImpl im
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
+ // AM is likely to receive duplicate TA_COMMIT_PENDINGs as the task
attempt
+ // will re-send the commit message until it doesn't encounter any
+ // IOException and succeeds in delivering the commit message.
+ // Ignoring the duplicate commit message is a short-term fix. In long
term,
+ // we need to make use of retry cache to help this and other MR protocol
+ // APIs that can be considered as @AtMostOnce.
+ .addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
+ TaskAttemptStateInternal.COMMIT_PENDING,
+ TaskAttemptEventType.TA_COMMIT_PENDING)
// Transitions from SUCCESS_CONTAINER_CLEANUP state
// kill and cleanup the container
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java?rev=1603338&r1=1603337&r2=1603338&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
Wed Jun 18 00:51:39 2014
@@ -112,6 +112,15 @@ public class TestMRApp {
//wait for first attempt to commit pending
app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING);
+ //re-send the commit pending signal to the task
+ app.getContext().getEventHandler().handle(
+ new TaskAttemptEvent(
+ attempt.getID(),
+ TaskAttemptEventType.TA_COMMIT_PENDING));
+
+ //the task attempt should be still at COMMIT_PENDING
+ app.waitForState(attempt, TaskAttemptState.COMMIT_PENDING);
+
//send the done signal to the task
app.getContext().getEventHandler().handle(
new TaskAttemptEvent(
