Author: tgraves Date: Wed Apr 4 14:18:55 2012 New Revision: 1309403 URL: http://svn.apache.org/viewvc?rev=1309403&view=rev Log: merge -r 1309400:1309401 from trunk. FIXES: MAPREDUCE-3672
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Apr 4 14:18:55 2012 @@ -141,6 +141,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4020. Web services returns incorrect JSON for deep queue tree (Anupam Seth via tgraves) + MAPREDUCE-3672. Killed maps shouldn't be counted towards + JobCounter.NUM_FAILED_MAPS. (Anupam Seth via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Apr 4 14:18:55 2012 @@ -988,6 +988,23 @@ public abstract class TaskAttemptImpl im } return jce; } + + private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( + TaskAttemptImpl taskAttempt) { + TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); + JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); + + long slotMillisIncrement = computeSlotMillis(taskAttempt); + + if (taskType == TaskType.MAP) { + jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + } else { + jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + } + return jce; + } private static TaskAttemptUnsuccessfulCompletionEvent @@ -1214,8 +1231,13 @@ public abstract class TaskAttemptImpl im TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, finalState); - taskAttempt.eventHandler + if(finalState == TaskAttemptState.FAILED) { + taskAttempt.eventHandler .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + } else if(finalState == TaskAttemptState.KILLED) { + taskAttempt.eventHandler + .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); + } taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); } else { @@ -1441,7 +1463,7 @@ public abstract class TaskAttemptImpl im taskAttempt.setFinishTime(); if (taskAttempt.getLaunchTime() != 0) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, TaskAttemptState.KILLED); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java Wed Apr 4 14:18:55 2012 @@ -27,6 +27,8 @@ import org.apache.hadoop.classification. public enum JobCounter { NUM_FAILED_MAPS, NUM_FAILED_REDUCES, + NUM_KILLED_MAPS, + NUM_KILLED_REDUCES, TOTAL_LAUNCHED_MAPS, TOTAL_LAUNCHED_REDUCES, OTHER_LOCAL_MAPS, Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties Wed Apr 4 14:18:55 2012 @@ -16,6 +16,8 @@ CounterGroupName= Job C NUM_FAILED_MAPS.name= Failed map tasks NUM_FAILED_REDUCES.name= Failed reduce tasks +NUM_KILLED_MAPS.name= Killed map tasks +NUM_KILLED_REDUCES.name= Killed reduce tasks TOTAL_LAUNCHED_MAPS.name= Launched map tasks TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks OTHER_LOCAL_MAPS.name= Other local map tasks Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java Wed Apr 4 14:18:55 2012 @@ -195,6 +195,8 @@ public class TestJobCleanup { RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); job.waitForCompletion(); + Counters counters = job.getCounters(); + assertTrue("No. of failed maps should be 1",counters.getCounter(JobCounter.NUM_FAILED_MAPS) == 1); if (fileName != null) { Path testFile = new Path(outDir, fileName); @@ -240,6 +242,9 @@ public class TestJobCleanup { job.killJob(); // kill the job job.waitForCompletion(); // wait for the job to complete + + counters = job.getCounters(); + assertTrue("No. of killed maps should be 1", counters.getCounter(JobCounter.NUM_KILLED_MAPS) == 1); if (fileName != null) { Path testFile = new Path(outDir, fileName); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java?rev=1309403&r1=1309402&r2=1309403&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java Wed Apr 4 14:18:55 2012 @@ -234,8 +234,10 @@ public class TestSpeculativeExecution { .getValue()); Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES) .getValue()); - Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_FAILED_MAPS) + Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS) .getValue()); + Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_KILLED_MAPS) + .getValue()); /*---------------------------------------------------------------------- * Test that Reducer speculates if REDUCE_SPECULATIVE is true and