[ https://issues.apache.org/jira/browse/MAPREDUCE-6724?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15384827#comment-15384827 ]
Gera Shegalov edited comment on MAPREDUCE-6724 at 7/19/16 8:49 PM: ------------------------------------------------------------------- As for test I was thinking of a smaller change to the existing test along the lines: {code} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index 1c0d25b..5930139 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -289,6 +289,19 @@ public void testLargeMemoryLimits() throws Exception { final long maxInMemReduce = mgr.getMaxInMemReduceLimit(); assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce, maxInMemReduce > Integer.MAX_VALUE); + + // verifyReserveLargeMapOutput(mgr, (long)Integer.MAX_VALUE); + verifyReserveLargeMapOutput(mgr, (long)Integer.MAX_VALUE + 1); + } + + private void verifyReserveLargeMapOutput(MergeManagerImpl<Text, Text> mgr, + long size) throws IOException { + final TaskAttemptID mapId = TaskAttemptID.forName("attempt_0_1_m_1_1"); + final MapOutput<Text, Text> mapOutput = mgr.reserve(mapId, size, 1); + assertEquals("Shuffled bytes: " + size, + size <= Integer.MAX_VALUE ? "MEMORY" : "DISK", + mapOutput.getDescription()); + mgr.unreserve(size); } {code} In terms of the production code, testing this I noticed that {code}mapreduce.task.reduce.MergeManagerImpl#canShuffleToMemory{code} having '<' does not look right, and should be '<='. I suggest we fix it in this JIRA as well. Probably a good idea to inline this simple private one-liner. was (Author: jira.shegalov): As for test I was thinking of a smaller change to the existing test along the lines: {code} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index 1c0d25b..f4e6680 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -289,6 +289,20 @@ public void testLargeMemoryLimits() throws Exception { final long maxInMemReduce = mgr.getMaxInMemReduceLimit(); assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce, maxInMemReduce > Integer.MAX_VALUE); + + verifyReserveLargeMapOutput(mgr, Integer.MAX_VALUE); + verifyReserveLargeMapOutput(mgr, Integer.MAX_VALUE + 1); + + } + + private void verifyReserveLargeMapOutput(MergeManagerImpl<Text, Text> mgr, + long size) throws IOException { + final TaskAttemptID mapId = TaskAttemptID.forName("attempt_0_1_m_1_1"); + final MapOutput<Text, Text> mapOutput = mgr.reserve(mapId, size, 1); + assertEquals("Shuffled bytes: " + size, + size <= Integer.MAX_VALUE ? "MEMORY" : "DISK", + mapOutput.getDescription()); + mgr.unreserve(size); } {code} In terms of the production code, testing this I noticed that {code}mapreduce.task.reduce.MergeManagerImpl#canShuffleToMemory{code} having '<' does not look right, and should be '<='. I suggest we fix it in this JIRA as well. Probably a good idea to inline this simple private one-liner. > Unsafe conversion from long to int in MergeManagerImpl.unconditionalReserve() > ----------------------------------------------------------------------------- > > Key: MAPREDUCE-6724 > URL: https://issues.apache.org/jira/browse/MAPREDUCE-6724 > Project: Hadoop Map/Reduce > Issue Type: Bug > Components: mrv2 > Reporter: Haibo Chen > Assignee: Haibo Chen > Attachments: mapreduce6724.001.patch, mapreduce6724.002.patch, > mapreduce6724.003.patch, mapreduce6724.004.patch, mapreduce6724.005.patch > > > When shuffle is done in memory, MergeManagerImpl converts the requested size > to an int to allocate an instance of InMemoryMapOutput. This results in an > overflow if the requested size is bigger than Integer.MAX_VALUE and > eventually causes the reducer to fail. -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: mapreduce-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: mapreduce-issues-h...@hadoop.apache.org