Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 167e54ed3 -> 07b702d9d


MAPREDUCE-6558. multibyte delimiters with compressed input files generate 
duplicate records. Contributed by Wilfred Spiegelenburg
(cherry picked from commit 9227dfc25f373a99cb66ad7d6bacef8dcf336f77)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07b702d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07b702d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07b702d9

Branch: refs/heads/branch-2.6
Commit: 07b702d9d520ac8cd6d12eb2a07dd371274f028e
Parents: 167e54e
Author: Jason Lowe <jl...@apache.org>
Authored: Fri May 13 14:38:57 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri May 13 14:38:57 2016 +0000

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |   3 ++
 .../lib/input/CompressedSplitLineReader.java    |   5 ++++
 .../hadoop/mapred/TestLineRecordReader.java     |  29 +++++++++++++++++++
 .../lib/input/TestLineRecordReader.java         |  29 +++++++++++++++++++
 .../compressedMultibyteDelimiter.txt.bz2        | Bin 0 -> 1096 bytes
 5 files changed, 66 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/07b702d9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt 
b/hadoop-mapreduce-project/CHANGES.txt
index 41eab79..85898cd 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -33,6 +33,9 @@ Release 2.6.5 - UNRELEASED
     MAPREDUCE-6689. MapReduce job can infinitely increase number of reducer
     resource requests (Wangda Tan via jlowe)
 
+    MAPREDUCE-6558. multibyte delimiters with compressed input files generate
+    duplicate records (Wilfred Spiegelenburg via jlowe)
+
 Release 2.6.4 - 2016-02-11
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/07b702d9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
index ef51f5c..9d0e949 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CompressedSplitLineReader.java
@@ -165,4 +165,9 @@ public class CompressedSplitLineReader extends 
SplitLineReader {
   public boolean needAdditionalRecordAfterSplit() {
     return !finished && needAdditionalRecord;
   }
+
+  @Override
+  protected void unsetNeedAdditionalRecordAfterSplit() {
+    needAdditionalRecord = false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/07b702d9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
index cb733bd..aa2bec1 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
@@ -589,4 +589,33 @@ public class TestLineRecordReader {
     assertFalse(reader.next(key, value));
     assertEquals(12, reader.getPos());
   }
+
+  @Test
+  public void testBzipWithMultibyteDelimiter() throws IOException {
+    String testFileName = "compressedMultibyteDelimiter.txt.bz2";
+    // firstSplitLength < (headers + blockMarker) will pass always since no
+    // records will be read (in the test file that is byte 0..9)
+    // firstSplitlength > (compressed file length - one compressed block
+    // size + 1) will also always pass since the second split will be empty
+    // (833 bytes is the last block start in the used data file)
+    int firstSplitLength = 100;
+    URL testFileUrl = getClass().getClassLoader().getResource(testFileName);
+    assertNotNull("Cannot find " + testFileName, testFileUrl);
+    File testFile = new File(testFileUrl.getFile());
+    long testFileSize = testFile.length();
+    Path testFilePath = new Path(testFile.getAbsolutePath());
+    assertTrue("Split size is smaller than header length",
+        firstSplitLength > 9);
+    assertTrue("Split size is larger than compressed file size " +
+        testFilePath, testFileSize > firstSplitLength);
+
+    Configuration conf = new Configuration();
+    conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+        LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+
+    String delimiter = "<E-LINE>\r\r\n";
+    conf.set("textinputformat.record.delimiter", delimiter);
+    testSplitRecordsForFile(conf, firstSplitLength, testFileSize,
+        testFilePath);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/07b702d9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
index d31f030..b8d7c09 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
@@ -572,4 +572,33 @@ public class TestLineRecordReader {
     // Key should be 12 right after "123456789\r\r\n"
     assertEquals(12, key.get());
   }
+
+  @Test
+  public void testBzipWithMultibyteDelimiter() throws IOException {
+    String testFileName = "compressedMultibyteDelimiter.txt.bz2";
+    // firstSplitLength < (headers + blockMarker) will pass always since no
+    // records will be read (in the test file that is byte 0..9)
+    // firstSplitlength > (compressed file length - one compressed block
+    // size + 1) will also always pass since the second split will be empty
+    // (833 bytes is the last block start in the used data file)
+    int firstSplitLength = 100;
+    URL testFileUrl = getClass().getClassLoader().getResource(testFileName);
+    assertNotNull("Cannot find " + testFileName, testFileUrl);
+    File testFile = new File(testFileUrl.getFile());
+    long testFileSize = testFile.length();
+    Path testFilePath = new Path(testFile.getAbsolutePath());
+    assertTrue("Split size is smaller than header length",
+        firstSplitLength > 9);
+    assertTrue("Split size is larger than compressed file size " +
+        testFilePath, testFileSize > firstSplitLength);
+
+    Configuration conf = new Configuration();
+    conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+        LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+
+    String delimiter = "<E-LINE>\r\r\n";
+    conf.set("textinputformat.record.delimiter", delimiter);
+    testSplitRecordsForFile(conf, firstSplitLength, testFileSize,
+        testFilePath);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/07b702d9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
----------------------------------------------------------------------
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
new file mode 100644
index 0000000..f8e178f
Binary files /dev/null and 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/compressedMultibyteDelimiter.txt.bz2
 differ


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

Reply via email to