Author: acmurthy Date: Mon Sep 16 14:19:45 2013 New Revision: 1523662 URL: http://svn.apache.org/r1523662 Log: Merge -c 1523660 from trunk to branch-2 to fix MAPREDUCE-5493. Cleanup in-memory & on-disk segments to prevent leak on shuffle completion. Contributed by Jason Lowe.
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1523662&r1=1523661&r2=1523662&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/CHANGES.txt Mon Sep 16 14:19:45 2013 @@ -92,6 +92,9 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5164. mapred job and queue commands omit HADOOP_CLIENT_OPTS (Nemon Lou via devaraj) + MAPREDUCE-5493. Cleanup in-memory & on-disk segments to prevent leak on + shuffle completion. (jlowe via acmurthy) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java?rev=1523662&r1=1523661&r2=1523662&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java Mon Sep 16 14:19:45 2013 @@ -355,8 +355,11 @@ public class MergeManagerImpl<K, V> impl List<InMemoryMapOutput<K, V>> memory = new ArrayList<InMemoryMapOutput<K, V>>(inMemoryMergedMapOutputs); + inMemoryMergedMapOutputs.clear(); memory.addAll(inMemoryMapOutputs); + inMemoryMapOutputs.clear(); List<CompressAwarePath> disk = new ArrayList<CompressAwarePath>(onDiskMapOutputs); + onDiskMapOutputs.clear(); return finalMerge(jobConf, rfs, memory, disk); } Modified: hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java?rev=1523662&r1=1523661&r2=1523662&view=diff ============================================================================== --- hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java (original) +++ hadoop/common/branches/branch-2.1-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java Mon Sep 16 14:19:45 2013 @@ -82,7 +82,7 @@ public class TestMerger { } @Test - public void testInMemoryMerger() throws IOException { + public void testInMemoryMerger() throws Throwable { JobID jobId = new JobID("a", 0); TaskAttemptID reduceId = new TaskAttemptID( new TaskID(jobId, TaskType.REDUCE, 0), 0); @@ -132,6 +132,11 @@ public class TestMerger { readOnDiskMapOutput(conf, fs, outPath, keys, values); Assert.assertEquals(keys, Arrays.asList("apple", "banana", "carrot")); Assert.assertEquals(values, Arrays.asList("disgusting", "pretty good", "delicious")); + + mergeManager.close(); + Assert.assertEquals(0, mergeManager.inMemoryMapOutputs.size()); + Assert.assertEquals(0, mergeManager.inMemoryMergedMapOutputs.size()); + Assert.assertEquals(0, mergeManager.onDiskMapOutputs.size()); } private byte[] writeMapOutput(Configuration conf, Map<String, String> keysToValues)