This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 3e951120a47 HBASE-29158 Unknown checksum type code exception occurred
while reading HFileBlock (#6740)
3e951120a47 is described below
commit 3e951120a47f1b9bf44fd31dafb32a158b6209f4
Author: xiaguanglei <[email protected]>
AuthorDate: Fri Mar 21 17:01:43 2025 +0800
HBASE-29158 Unknown checksum type code exception occurred while reading
HFileBlock (#6740)
Co-authored-by: xiaguanglei <[email protected]>
Signed-off-by: Nick Dimiduk <[email protected]>
---
.../apache/hadoop/hbase/io/hfile/HFileBlock.java | 44 +++++++
.../io/hfile/TestHFileBlockHeaderCorruption.java | 140 ++++++++++++++++++++-
2 files changed, 183 insertions(+), 1 deletion(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
index 91678d58b6e..3025022688a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
@@ -1587,6 +1587,24 @@ public class HFileBlock implements Cacheable {
}
}
+ /**
+ * Check that checksumType on {@code headerBuf} read from a block header
seems reasonable,
+ * within the known value range.
+ * @return {@code true} if the headerBuf is safe to proceed, {@code false}
otherwise.
+ */
+ private boolean checkCheckSumTypeOnHeaderBuf(ByteBuff headerBuf) {
+ if (headerBuf == null) {
+ return true;
+ }
+ byte b = headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX);
+ for (ChecksumType t : ChecksumType.values()) {
+ if (t.getCode() == b) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Check that {@code value} read from a block header seems reasonable,
within a large margin of
* error.
@@ -1742,6 +1760,21 @@ public class HFileBlock implements Cacheable {
onDiskSizeWithHeader = getOnDiskSizeWithHeader(headerBuf,
checksumSupport);
}
+ // Inspect the header's checksumType for known valid values. If we don't
find such a value,
+ // assume that the bytes read are corrupted.We will clear the cached
value and roll back to
+ // HDFS checksum
+ if (!checkCheckSumTypeOnHeaderBuf(headerBuf)) {
+ if (verifyChecksum) {
+ invalidateNextBlockHeader();
+ span.addEvent("Falling back to HDFS checksumming.",
attributesBuilder.build());
+ return null;
+ } else {
+ throw new IOException(
+ "Unknown checksum type code " +
headerBuf.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX)
+ + "for file " + pathName + ", the headerBuf of HFileBlock may
corrupted.");
+ }
+ }
+
// The common case is that onDiskSizeWithHeader was produced by a read
without checksum
// validation, so give it a sanity check before trying to use it.
if (!checkOnDiskSizeWithHeader(onDiskSizeWithHeader)) {
@@ -1885,6 +1918,17 @@ public class HFileBlock implements Cacheable {
if (!fileContext.isUseHBaseChecksum()) {
return false;
}
+
+ // If the checksumType of the read block header is incorrect, it
indicates that the block is
+ // corrupted and can be directly rolled back to HDFS checksum
verification
+ if (!checkCheckSumTypeOnHeaderBuf(data)) {
+ HFile.LOG.warn(
+ "HBase checksumType verification failed for file {} at offset {}
filesize {}"
+ + " checksumType {}. Retrying read with HDFS checksums turned
on...",
+ pathName, offset, fileSize,
data.get(HFileBlock.Header.CHECKSUM_TYPE_INDEX));
+ return false;
+ }
+
return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize);
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java
index e9cd8260e9b..2f9147e8619 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockHeaderCorruption.java
@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.startsWith;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -55,6 +56,7 @@ import org.apache.hadoop.hbase.nio.ByteBuff;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.TypeSafeMatcher;
@@ -93,6 +95,141 @@ public class TestHFileBlockHeaderCorruption {
ruleChain = RuleChain.outerRule(testName).around(hFileTestRule);
}
+ @Test
+ public void testChecksumTypeCorruptionFirstBlock() throws Exception {
+ HFileBlockChannelPosition firstBlock = null;
+ try {
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ assertTrue(it.hasNext());
+ firstBlock = it.next();
+ }
+
+ Corrupter c = new Corrupter(firstBlock);
+
+ logHeader(firstBlock);
+
+ // test corrupted HFileBlock with unknown checksumType code -1
+ c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new
byte[] { -1 }));
+ logHeader(firstBlock);
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ CountingConsumer consumer = new CountingConsumer(it);
+ try {
+ consumer.readFully();
+ fail();
+ } catch (Exception e) {
+ assertThat(e, new
IsThrowableMatching().withInstanceOf(IOException.class)
+ .withMessage(startsWith("Unknown checksum type code")));
+ }
+ assertEquals(0, consumer.getItemsRead());
+ }
+
+ // valid checksumType code test
+ for (ChecksumType t : ChecksumType.values()) {
+ testValidChecksumTypeReadBlock(t.getCode(), c, firstBlock);
+ }
+
+ c.restore();
+ // test corrupted HFileBlock with unknown checksumType code 3
+ c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new
byte[] { 3 }));
+ logHeader(firstBlock);
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ CountingConsumer consumer = new CountingConsumer(it);
+ try {
+ consumer.readFully();
+ fail();
+ } catch (Exception e) {
+ assertThat(e, new
IsThrowableMatching().withInstanceOf(IOException.class)
+ .withMessage(startsWith("Unknown checksum type code")));
+ }
+ assertEquals(0, consumer.getItemsRead());
+ }
+ } finally {
+ if (firstBlock != null) {
+ firstBlock.close();
+ }
+ }
+ }
+
+ @Test
+ public void testChecksumTypeCorruptionSecondBlock() throws Exception {
+ HFileBlockChannelPosition secondBlock = null;
+ try {
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ assertTrue(it.hasNext());
+ it.next();
+ assertTrue(it.hasNext());
+ secondBlock = it.next();
+ }
+
+ Corrupter c = new Corrupter(secondBlock);
+
+ logHeader(secondBlock);
+ // test corrupted HFileBlock with unknown checksumType code -1
+ c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new
byte[] { -1 }));
+ logHeader(secondBlock);
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ CountingConsumer consumer = new CountingConsumer(it);
+ try {
+ consumer.readFully();
+ fail();
+ } catch (Exception e) {
+ assertThat(e, new
IsThrowableMatching().withInstanceOf(RuntimeException.class)
+ .withMessage(startsWith("Unknown checksum type code")));
+ }
+ assertEquals(1, consumer.getItemsRead());
+ }
+
+ // valid checksumType code test
+ for (ChecksumType t : ChecksumType.values()) {
+ testValidChecksumTypeReadBlock(t.getCode(), c, secondBlock);
+ }
+
+ c.restore();
+ // test corrupted HFileBlock with unknown checksumType code 3
+ c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX, ByteBuffer.wrap(new
byte[] { 3 }));
+ logHeader(secondBlock);
+ try (HFileBlockChannelPositionIterator it =
+ new HFileBlockChannelPositionIterator(hFileTestRule)) {
+ CountingConsumer consumer = new CountingConsumer(it);
+ try {
+ consumer.readFully();
+ fail();
+ } catch (Exception e) {
+ assertThat(e, new
IsThrowableMatching().withInstanceOf(RuntimeException.class)
+ .withMessage(startsWith("Unknown checksum type code")));
+ }
+ assertEquals(1, consumer.getItemsRead());
+ }
+ } finally {
+ if (secondBlock != null) {
+ secondBlock.close();
+ }
+ }
+ }
+
+ public void testValidChecksumTypeReadBlock(byte checksumTypeCode, Corrupter
c,
+ HFileBlockChannelPosition testBlock) throws IOException {
+ c.restore();
+ c.write(HFileBlock.Header.CHECKSUM_TYPE_INDEX,
+ ByteBuffer.wrap(new byte[] { checksumTypeCode }));
+ logHeader(testBlock);
+ try (
+ HFileBlockChannelPositionIterator it = new
HFileBlockChannelPositionIterator(hFileTestRule)) {
+ CountingConsumer consumer = new CountingConsumer(it);
+ try {
+ consumer.readFully();
+ } catch (Exception e) {
+ fail("test fail: valid checksumType are not executing properly");
+ }
+ assertNotEquals(0, consumer.getItemsRead());
+ }
+ }
+
@Test
public void testOnDiskSizeWithoutHeaderCorruptionFirstBlock() throws
Exception {
HFileBlockChannelPosition firstBlock = null;
@@ -331,7 +468,8 @@ public class TestHFileBlockHeaderCorruption {
try {
reader = HFile.createReader(hfs, hfsPath, CacheConfig.DISABLED, true,
conf);
HFileBlock.FSReader fsreader = reader.getUncachedBlockReader();
- iter = fsreader.blockRange(0, hfs.getFileStatus(hfsPath).getLen());
+ // The read block offset cannot out of the range:0,loadOnOpenDataOffset
+ iter = fsreader.blockRange(0,
reader.getTrailer().getLoadOnOpenDataOffset());
} catch (IOException e) {
if (reader != null) {
closeQuietly(reader::close);
