Author: jlowe Date: Mon May 13 20:54:58 2013 New Revision: 1482100 URL: http://svn.apache.org/r1482100 Log: MAPREDUCE-5211. Reducer intermediate files can collide during merge. Contributed by Jason Lowe
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1482100&r1=1482099&r2=1482100&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon May 13 20:54:58 2013 @@ -24,6 +24,9 @@ Release 0.23.8 - UNRELEASED MAPREDUCE-5147. Maven build should create hadoop-mapreduce-client-app-VERSION.jar directly (Robert Parker via tgraves) + MAPREDUCE-5211. Reducer intermediate files can collide during merge + (jlowe) + Release 0.23.7 - 2013-04-18 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?rev=1482100&r1=1482099&r2=1482100&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Mon May 13 20:54:58 2013 @@ -505,7 +505,8 @@ public class MergeManager<K, V> { } private class OnDiskMerger extends MergeThread<MapOutput<K,V>,K,V> { - + private int mergeCount = 0; + public OnDiskMerger(MergeManager<K, V> manager) { super(manager, ioSortFactor, exceptionReporter); setName("OnDiskMerger - Thread to merge on-disk map-outputs"); @@ -537,8 +538,10 @@ public class MergeManager<K, V> { ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum); // 2. Start the on-disk merge process + ++mergeCount; Path outputPath = - localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(), + localDirAllocator.getLocalPathForWrite( + new Path(reduceId.toString(), "diskmerge" + mergeCount).toString(), approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX); Writer<K,V> writer = new Writer<K,V>(jobConf, rfs, outputPath,