MAPREDUCE-6923. Optimize MapReduce Shuffle I/O for small partitions. Contributed by Robert Schmidtke.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac7d0604 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac7d0604 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac7d0604 Branch: refs/heads/YARN-1011 Commit: ac7d0604bc73c0925eff240ad9837e14719d57b7 Parents: b5c02f9 Author: Ravi Prakash <ravip...@altiscale.com> Authored: Wed Aug 9 15:39:52 2017 -0700 Committer: Ravi Prakash <ravip...@altiscale.com> Committed: Wed Aug 9 15:39:52 2017 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac7d0604/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index cb9b5e0..79045f9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -111,7 +111,10 @@ public class FadvisedFileRegion extends DefaultFileRegion { long trans = actualCount; int readSize; - ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize); + ByteBuffer byteBuffer = ByteBuffer.allocate( + Math.min( + this.shuffleBufferSize, + trans > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) trans)); while(trans > 0L && (readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org