This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 0ea11f2fc3a HBASE-28390 WAL value compression fails for cells with
large values (#5696)
0ea11f2fc3a is described below
commit 0ea11f2fc3a526b24c82248db0346d28bf20d12c
Author: Bryan Beaudreault <bbeaudrea...@apache.org>
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell <apurt...@apache.org>
---
.../aircompressor/TestWALCompressionLz4.java | 13 ---------
.../aircompressor/TestWALCompressionLzo.java | 14 ----------
.../aircompressor/TestWALCompressionSnappy.java | 13 ---------
.../aircompressor/TestWALCompressionZstd.java | 13 ---------
.../compress/brotli/TestWALCompressionBrotli.java | 13 ---------
.../io/compress/lz4/TestWALCompressionLz4.java | 14 ----------
.../compress/xerial/TestWALCompressionSnappy.java | 13 ---------
.../io/compress/xz/TestWALCompressionLzma.java | 13 ---------
.../io/compress/zstd/TestWALCompressionZstd.java | 14 ----------
.../hbase/regionserver/wal/CompressionContext.java | 17 ++++++++++++
...LDecompressionBoundedDelegatingInputStream.java | 16 ++++++-----
.../hadoop/hbase/wal/CompressedWALTestBase.java | 32 ++++++++++++++++++++--
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 ---------
.../wal/TestCompressedWALValueCompression.java | 13 ---------
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzo extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLzo extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzo.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionLzo extends
CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
index 72813bcbd65..93345b924fa 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionSnappy extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionSnappy extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
index 0f5c80ce269..3b367a2db9b 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionZstd extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
b/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
index e37276fed6d..3539c4280d9 100644
---
a/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
+++
b/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.brotli;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionBrotli extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionBrotli extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionBrotli.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionBrotli extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
index 81b5d943dc6..b79fe094de0 100644
---
a/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.lz4;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
index 7a2bbfe7b87..bb6dac1e452 100644
---
a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
+++
b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
@@ -22,7 +22,6 @@ import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -31,10 +30,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
@@ -43,9 +39,6 @@ public class TestWALCompressionSnappy extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
@@ -63,10 +56,4 @@ public class TestWALCompressionSnappy extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
b/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
index ee937230cd2..aa74926cb81 100644
---
a/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
+++
b/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.xz;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzma extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLzma extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzma.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLzma extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
index 55d61cf83ec..23a37ab8659 100644
---
a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
+++
b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.zstd;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionZstd extends
CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
index 633decab0a8..2481753dfb0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -129,9 +130,25 @@ public class CompressionContext {
} else {
lowerIn.reset(in, inLength);
IOUtils.readFully(compressedIn, outArray, outOffset, outLength);
+ // if the uncompressed size was larger than the configured buffer size
for the codec,
+ // the BlockCompressorStream will have left an extra 4 bytes hanging.
This represents a size
+ // for the next segment, and it should be 0. See HBASE-28390
+ if (lowerIn.available() == 4) {
+ int remaining = rawReadInt(lowerIn);
+ assert remaining == 0;
+ }
}
}
+ private int rawReadInt(InputStream in) throws IOException {
+ int b1 = in.read();
+ int b2 = in.read();
+ int b3 = in.read();
+ int b4 = in.read();
+ if ((b1 | b2 | b3 | b4) < 0) throw new EOFException();
+ return ((b1 << 24) + (b2 << 16) + (b3 << 8) + (b4 << 0));
+ }
+
public void clear() {
if (compressedOut != null) {
try {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
index 0f4fd78a0b8..5d876b97aa8 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
@@ -67,19 +66,22 @@ class WALDecompressionBoundedDelegatingInputStream extends
InputStream {
if (pos >= limit) {
return -1;
}
- int readLen = (int) Math.min(len, limit - pos);
- try {
- IOUtils.readFully(in, b, off, readLen);
- } catch (EOFException e) {
+ int toRead = (int) Math.min(len, limit - pos);
+ int readBytes = IOUtils.read(in, b, off, toRead);
+ // increase pos by however many we actually read
+ pos += readBytes;
+
+ if (readBytes != toRead) {
// This is trick here, we will always try to read enough bytes to fill
the buffer passed in,
// or we reach the end of this compression block, if there are not
enough bytes, we just
// return -1 to let the upper layer fail with EOF
// In WAL value decompression this is OK as if we can not read all the
data, we will finally
// get an EOF somewhere
- LOG.debug("Got EOF while we want to read {} bytes from stream", readLen,
e);
+ LOG.debug("Got EOF while we want to read {} bytes from stream, but only
read {}", toRead,
+ readBytes);
return -1;
}
- return readLen;
+ return toRead;
}
@Override
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
index 751e4dc88b3..2ff9223e74b 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
@@ -36,9 +36,13 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("checkstyle:innerassignment")
-public class CompressedWALTestBase {
+public abstract class CompressedWALTestBase {
+ private static final Logger LOG =
LoggerFactory.getLogger(CompressedWALTestBase.class);
protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
@@ -67,14 +71,36 @@ public class CompressedWALTestBase {
Arrays.fill(VALUE, off, (off += 1597), (byte) 'Q');
}
- public void doTest(TableName tableName) throws Exception {
+ @Test
+ public void test() throws Exception {
+ testForSize(1000);
+ }
+
+ @Test
+ public void testLarge() throws Exception {
+ testForSize(1024 * 1024);
+ }
+
+ private void testForSize(int size) throws Exception {
+ TableName tableName = TableName.valueOf(getClass().getSimpleName() +
"_testForSize_" + size);
+ doTest(tableName, size);
+ }
+
+ public void doTest(TableName tableName, int valueSize) throws Exception {
NavigableMap<byte[], Integer> scopes = new
TreeMap<>(Bytes.BYTES_COMPARATOR);
scopes.put(tableName.getName(), 0);
RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
final int total = 1000;
final byte[] row = Bytes.toBytes("row");
final byte[] family = Bytes.toBytes("family");
- final byte[] value = VALUE;
+ final byte[] value = new byte[valueSize];
+
+ int offset = 0;
+ while (offset + VALUE.length < value.length) {
+ System.arraycopy(VALUE, 0, value, offset, VALUE.length);
+ offset += VALUE.length;
+ }
+
final WALFactory wals =
new WALFactory(TEST_UTIL.getConfiguration(),
tableName.getNameAsString());
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
index d1275650793..faf244b5db7 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
@@ -20,16 +20,12 @@ package org.apache.hadoop.hbase.wal;
import java.util.Arrays;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
@@ -43,9 +39,6 @@ public class TestCompressedWAL extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestCompressedWAL.class);
- @Rule
- public TestName name = new TestName();
-
@Parameter
public String walProvider;
@@ -66,10 +59,4 @@ public class TestCompressedWAL extends CompressedWALTestBase
{
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
index ca4c8dd8c53..598fd9f7a9e 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import
org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -50,9 +46,6 @@ public class TestCompressedWALValueCompression extends
CompressedWALTestBase {
return HBaseCommonTestingUtil.COMPRESSION_ALGORITHMS_PARAMETERIZED;
}
- @Rule
- public TestName name = new TestName();
-
private final Compression.Algorithm compression;
public TestCompressedWALValueCompression(Compression.Algorithm algo) {
@@ -72,10 +65,4 @@ public class TestCompressedWALValueCompression extends
CompressedWALTestBase {
public void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
}
