This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 0e331484d01 HBASE-30000 testCompactionWithCorruptBlock fails on
branch-2.5 (#7962)
0e331484d01 is described below
commit 0e331484d01f634f2cae9c41845a050ca3e882ce
Author: Nick Dimiduk <[email protected]>
AuthorDate: Mon Mar 23 12:35:32 2026 +0100
HBASE-30000 testCompactionWithCorruptBlock fails on branch-2.5 (#7962)
Signed-off-by: Charles Connell <[email protected]>
Signed-off-by: Xiao Liu <[email protected]>
Signed-off-by: Duo Zhang <[email protected]>
---
.../hadoop/hbase/regionserver/TestCompaction.java | 40 ++++++++++++++++++++--
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index 6427095c981..45396e8fa2f 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -40,6 +40,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -52,6 +54,7 @@ import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -91,6 +94,7 @@ import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -154,7 +158,10 @@ public class TestCompaction {
hcd.setMaxVersions(65536);
this.htd.addFamily(hcd);
}
- if (name.getMethodName().equals("testCompactionWithCorruptBlock")) {
+ if (
+ name.getMethodName().equals("testCompactionWithCorruptBlock")
+ || name.getMethodName().equals("generateHFileForCorruptBlockTest")
+ ) {
UTIL.getConfiguration().setBoolean("hbase.hstore.validate.read_fully",
true);
HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
hcd.setCompressionType(Compression.Algorithm.GZ);
@@ -379,9 +386,38 @@ public class TestCompaction {
assertTrue(fs.exists(tmpPath));
}
+ /**
+ * Generates the HFile used by {@link #testCompactionWithCorruptBlock()}.
Run this method to
+ * regenerate the test resource file after changes to the HFile format. The
output file must then
+ * be hand-edited to corrupt the first data block (zero out the GZip magic
bytes at offset 33)
+ * before being placed into the test resources directory.
+ */
+ @Ignore("Not a test; utility for regenerating testCompactionWithCorruptBlock
resource file")
+ @Test
+ public void generateHFileForCorruptBlockTest() throws Exception {
+ createStoreFile(r, Bytes.toString(FAMILY));
+ createStoreFile(r, Bytes.toString(FAMILY));
+ HStore store = r.getStore(FAMILY);
+
+ Collection<HStoreFile> storeFiles = store.getStorefiles();
+ DefaultCompactor tool = (DefaultCompactor)
store.storeEngine.getCompactor();
+ CompactionRequestImpl request = new CompactionRequestImpl(storeFiles);
+ List<Path> paths = tool.compact(request,
NoLimitThroughputController.INSTANCE, null);
+
+ FileSystem fs = store.getFileSystem();
+ Path hfilePath = paths.get(0);
+ File outFile = new File("/tmp/TestCompaction_HFileWithCorruptBlock.gz");
+ try (InputStream in = fs.open(hfilePath);
+ GZIPOutputStream gzOut = new GZIPOutputStream(new
FileOutputStream(outFile))) {
+ IOUtils.copyBytes(in, gzOut, 4096);
+ }
+ LoggerFactory.getLogger(TestCompaction.class)
+ .info("Wrote HFile to {}. Now hex-edit offset 33 (0x21): zero out bytes
1f 8b.", outFile);
+ }
+
/**
* This test uses a hand-modified HFile, which is loaded in from the
resources' path. That file
- * was generated from the test support code in this class and then edited to
corrupt the
+ * was generated from {@link #generateHFileForCorruptBlockTest()} and then
edited to corrupt the
* GZ-encoded block by zeroing-out the first two bytes of the GZip header,
the "standard
* declaration" of {@code 1f 8b}, found at offset 33 in the file. I'm not
sure why, but it seems
* that in this test context we do not enforce CRC checksums. Thus, this
corruption manifests in
