MAPREDUCE-6063. Correct spill size calculation for spills wrapping the circular buffer. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a18424e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a18424e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a18424e7 Branch: refs/heads/MR-2841 Commit: a18424e724982d01e0191ca7db713da1d43d1e4d Parents: d9a03e2 Author: Chris Douglas <cdoug...@apache.org> Authored: Wed Sep 3 14:32:35 2014 -0700 Committer: Chris Douglas <cdoug...@apache.org> Committed: Wed Sep 3 17:01:59 2014 -0700 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/mapred/MapTask.java | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a18424e7/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 63bc08b..0ee0731 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -267,6 +267,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5931. Validate SleepJob command line parameters (Gera Shegalov via jlowe) + MAPREDUCE-6063. Correct spill size calculation for spills wrapping the + circular buffer. (zhihai xu via cdouglas) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/a18424e7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index dfcbe09..75b4141 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -1575,9 +1575,7 @@ public class MapTask extends Task { InterruptedException { //approximate the length of the output file to be the length of the //buffer + header lengths for the partitions - final long size = (bufend >= bufstart - ? bufend - bufstart - : (bufvoid - bufend) + bufstart) + + final long size = distanceTo(bufstart, bufend, bufvoid) + partitions * APPROX_HEADER_LENGTH; FSDataOutputStream out = null; try {
