Author: tgraves Date: Mon Jun 11 13:57:15 2012 New Revision: 1348848 URL: http://svn.apache.org/viewvc?rev=1348848&view=rev Log: merge -r 1348845:1348846 from trunk. FIXES: MAPREDUCE-3927
Added: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/ - copied from r1348846, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/ - copied from r1348846, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java - copied unchanged from r1348846, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestShuffleScheduler.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1348848&r1=1348847&r2=1348848&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Jun 11 13:57:15 2012 @@ -454,6 +454,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3842. Stop webpages from automatic refreshing (tgraves) + MAPREDUCE-3927. Shuffle hang when set map.failures.percent + (Bhallamudi Venkata Siva Kamesh via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1348848&r1=1348847&r2=1348848&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Mon Jun 11 13:57:15 2012 @@ -590,9 +590,9 @@ public class JobImpl implements org.apac float reduceProgress = 0f; for (Task task : this.tasks.values()) { if (task.getType() == TaskType.MAP) { - mapProgress += task.getProgress(); + mapProgress += (task.isFinished() ? 1f : task.getProgress()); } else { - reduceProgress += task.getProgress(); + reduceProgress += (task.isFinished() ? 1f : task.getProgress()); } } if (this.numMapTasks != 0) { Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java?rev=1348848&r1=1348847&r2=1348848&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleScheduler.java Mon Jun 11 13:57:15 2012 @@ -137,24 +137,26 @@ class ShuffleScheduler<K,V> { // update the status totalBytesShuffledTillNow += bytes; - float mbs = (float) totalBytesShuffledTillNow / (1024 * 1024); - int mapsDone = totalMaps - remainingMaps; - long secsSinceStart = - (System.currentTimeMillis()-startTime)/1000+1; - - float transferRate = mbs/secsSinceStart; - progress.set((float) mapsDone / totalMaps); - String statusString = mapsDone + " / " + totalMaps + " copied."; - status.setStateString(statusString); - progress.setStatus("copy(" + mapsDone + " of " + totalMaps - + " at " + - mbpsFormat.format(transferRate) + " MB/s)"); - + updateStatus(); reduceShuffleBytes.increment(bytes); lastProgressTime = System.currentTimeMillis(); - LOG.debug("map " + mapId + " done " + statusString); + LOG.debug("map " + mapId + " done " + status.getStateString()); } } + + private void updateStatus() { + float mbs = (float) totalBytesShuffledTillNow / (1024 * 1024); + int mapsDone = totalMaps - remainingMaps; + long secsSinceStart = (System.currentTimeMillis() - startTime) / 1000 + 1; + + float transferRate = mbs / secsSinceStart; + progress.set((float) mapsDone / totalMaps); + String statusString = mapsDone + " / " + totalMaps + " copied."; + status.setStateString(statusString); + + progress.setStatus("copy(" + mapsDone + " of " + totalMaps + " at " + + mbpsFormat.format(transferRate) + " MB/s)"); + } public synchronized void copyFailed(TaskAttemptID mapId, MapHost host, boolean readError) { @@ -256,7 +258,13 @@ class ShuffleScheduler<K,V> { } public synchronized void tipFailed(TaskID taskId) { - finishedMaps[taskId.getId()] = true; + if (!finishedMaps[taskId.getId()]) { + finishedMaps[taskId.getId()] = true; + if (--remainingMaps == 0) { + notifyAll(); + } + updateStatus(); + } } public synchronized void addKnownMapOutput(String hostName,