This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 575d119ce2d HBASE-30000 testCompactionWithCorruptBlock fails on
branch-2.5 (#7962)
575d119ce2d is described below
commit 575d119ce2d00b0298820bc2cb5704f0031cd119
Author: Nick Dimiduk <[email protected]>
AuthorDate: Mon Mar 23 12:35:32 2026 +0100
HBASE-30000 testCompactionWithCorruptBlock fails on branch-2.5 (#7962)
Signed-off-by: Charles Connell <[email protected]>
Signed-off-by: Xiao Liu <[email protected]>
Signed-off-by: Duo Zhang <[email protected]>
---
.../hadoop/hbase/regionserver/TestCompaction.java | 40 ++++++++++++++++++++--
1 file changed, 38 insertions(+), 2 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index de0766b7dba..7f738db4f3c 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -40,6 +40,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -52,6 +54,7 @@ import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -93,6 +96,7 @@ import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -156,7 +160,10 @@ public class TestCompaction {
ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setMaxVersions(65536).build();
builder.setColumnFamily(familyDescriptor);
}
- if (name.getMethodName().equals("testCompactionWithCorruptBlock")) {
+ if (
+ name.getMethodName().equals("testCompactionWithCorruptBlock")
+ || name.getMethodName().equals("generateHFileForCorruptBlockTest")
+ ) {
UTIL.getConfiguration().setBoolean("hbase.hstore.validate.read_fully",
true);
ColumnFamilyDescriptor familyDescriptor =
ColumnFamilyDescriptorBuilder.newBuilder(FAMILY)
.setCompressionType(Compression.Algorithm.GZ).build();
@@ -382,9 +389,38 @@ public class TestCompaction {
assertTrue(fs.exists(tmpPath));
}
+ /**
+ * Generates the HFile used by {@link #testCompactionWithCorruptBlock()}.
Run this method to
+ * regenerate the test resource file after changes to the HFile format. The
output file must then
+ * be hand-edited to corrupt the first data block (zero out the GZip magic
bytes at offset 33)
+ * before being placed into the test resources directory.
+ */
+ @Ignore("Not a test; utility for regenerating testCompactionWithCorruptBlock
resource file")
+ @Test
+ public void generateHFileForCorruptBlockTest() throws Exception {
+ createStoreFile(r, Bytes.toString(FAMILY));
+ createStoreFile(r, Bytes.toString(FAMILY));
+ HStore store = r.getStore(FAMILY);
+
+ Collection<HStoreFile> storeFiles = store.getStorefiles();
+ DefaultCompactor tool = (DefaultCompactor)
store.storeEngine.getCompactor();
+ CompactionRequestImpl request = new CompactionRequestImpl(storeFiles);
+ List<Path> paths = tool.compact(request,
NoLimitThroughputController.INSTANCE, null);
+
+ FileSystem fs = store.getFileSystem();
+ Path hfilePath = paths.get(0);
+ File outFile = new File("/tmp/TestCompaction_HFileWithCorruptBlock.gz");
+ try (InputStream in = fs.open(hfilePath);
+ GZIPOutputStream gzOut = new GZIPOutputStream(new
FileOutputStream(outFile))) {
+ IOUtils.copyBytes(in, gzOut, 4096);
+ }
+ LoggerFactory.getLogger(TestCompaction.class)
+ .info("Wrote HFile to {}. Now hex-edit offset 33 (0x21): zero out bytes
1f 8b.", outFile);
+ }
+
/**
* This test uses a hand-modified HFile, which is loaded in from the
resources' path. That file
- * was generated from the test support code in this class and then edited to
corrupt the
+ * was generated from {@link #generateHFileForCorruptBlockTest()} and then
edited to corrupt the
* GZ-encoded block by zeroing-out the first two bytes of the GZip header,
the "standard
* declaration" of {@code 1f 8b}, found at offset 33 in the file. I'm not
sure why, but it seems
* that in this test context we do not enforce CRC checksums. Thus, this
corruption manifests in
