Author: jlowe Date: Mon Dec 9 23:42:36 2013 New Revision: 1549710 URL: http://svn.apache.org/r1549710 Log: svn merge -c 1549705 FIXES: MAPREDUCE-5656. bzip2 codec can drop records when reading data in splits. Contributed by Jason Lowe
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java?rev=1549710&r1=1549709&r2=1549710&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java Mon Dec 9 23:42:36 2013 @@ -175,6 +175,11 @@ public class LineReader implements Close } } + protected int fillBuffer(InputStream in, byte[] buffer, boolean inDelimiter) + throws IOException { + return in.read(buffer); + } + /** * Read a line terminated by one of CR, LF, or CRLF. */ @@ -208,7 +213,7 @@ public class LineReader implements Close if (prevCharCR) { ++bytesConsumed; //account for CR from previous read } - bufferLength = in.read(buffer); + bufferLength = fillBuffer(in, buffer, prevCharCR); if (bufferLength <= 0) { break; // EOF } @@ -296,7 +301,7 @@ public class LineReader implements Close int startPosn = bufferPosn; // Start from previous end position if (bufferPosn >= bufferLength) { startPosn = bufferPosn = 0; - bufferLength = in.read(buffer); + bufferLength = fillBuffer(in, buffer, ambiguousByteCount > 0); if (bufferLength <= 0) { str.append(recordDelimiterBytes, 0, ambiguousByteCount); break; // EOF