This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 39880539ec56 fix(storage-format): emit HBase-readable block-index keys
in the native HFile writer (#19071)
39880539ec56 is described below
commit 39880539ec56cb3166abb8f1f10851d341abccdb
Author: Y Ethan Guo <[email protected]>
AuthorDate: Tue Aug 11 03:04:29 2026 -0700
fix(storage-format): emit HBase-readable block-index keys in the native
HFile writer (#19071)
---
hudi-io/hfile_format.md | 10 +-
.../java/org/apache/hudi/io/hfile/HFileBlock.java | 42 ++++
.../org/apache/hudi/io/hfile/HFileDataBlock.java | 27 +-
.../apache/hudi/io/hfile/HFileRootIndexBlock.java | 15 +-
.../apache/hudi/io/hfile/TestHFileDataBlock.java | 79 ++++++
.../hudi/io/hfile/TestHFileReadCompatibility.java | 275 ++++++++++++++++-----
.../hudi/io/hfile/TestHFileRootIndexBlock.java | 80 ++++++
.../org/apache/hudi/io/hfile/TestHFileWriter.java | 114 ++++++++-
8 files changed, 549 insertions(+), 93 deletions(-)
diff --git a/hudi-io/hfile_format.md b/hudi-io/hfile_format.md
index 192c3d4313f8..df96148b9f10 100644
--- a/hudi-io/hfile_format.md
+++ b/hudi-io/hfile_format.md
@@ -197,7 +197,10 @@ Key:
- **Key Content Size**: 2 byte, short, size of the key content.
- **Key Content**: key content in byte array. In Hudi, we serialize the String
into byte array using UTF-8.
-- **Other Information**: other information of the key, which is not used by
Hudi.
+- **Other Information**: the remaining fields that complete the KeyValue key,
so it parses as a
+ standard KeyValue key (and is readable by an HBase HFile reader): 1-byte
column-family length
+ (`0`), 8-byte timestamp (`Long.MAX_VALUE`, the "latest" sentinel), and
1-byte key type (`Put` = 4).
+ Hudi's reader ignores these fields.
Value:
@@ -286,7 +289,10 @@ For Data Index, the "Key Bytes" part has the following
format (same as the key f
- **Key Content Size**: 2 byte, short, size of the key content.
- **Key Content**: key content in byte array. In Hudi, we encode the String
into bytes using UTF-8.
-- **Other Information**: other information of the key, which is not used by
Hudi.
+- **Other Information**: the remaining fields that complete the KeyValue key,
so it parses as a
+ standard KeyValue key (and is readable by an HBase HFile reader): 1-byte
column-family length
+ (`0`), 8-byte timestamp (`Long.MAX_VALUE`, the "latest" sentinel), and
1-byte key type (`Put` = 4).
+ Hudi's reader ignores these fields.
For Meta Index, the "Key Bytes" part is the byte array of the key of the Meta
Block.
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
index 56f75466dc61..996af2759cdd 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileBlock.java
@@ -36,6 +36,7 @@ import java.nio.ByteBuffer;
import static org.apache.hudi.io.hfile.DataSize.MAGIC_LENGTH;
import static org.apache.hudi.io.hfile.DataSize.SIZEOF_BYTE;
+import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT16;
import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT32;
import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT64;
import static org.apache.hudi.io.util.IOUtils.readInt;
@@ -56,6 +57,12 @@ public abstract class HFileBlock {
static final int CHECKSUM_SIZE = SIZEOF_INT32;
private static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+ // Hudi does not set a version timestamp on key-value pairs, so the latest
timestamp is used.
+ private static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
+ // Key type is constant Put (4) in Hudi.
+ private static final byte KEY_TYPE_PUT = (byte) 4;
+ // KeyValue key suffix beyond the row: column-family length (1) + timestamp
(8) + type (1).
+ private static final int KEY_METADATA_SUFFIX_LENGTH = SIZEOF_BYTE +
SIZEOF_INT64 + SIZEOF_BYTE;
static class Header {
// Format of header is:
@@ -318,6 +325,41 @@ public abstract class HFileBlock {
throw new HoodieException("Only NULL checksum type is supported");
}
+ /**
+ * Returns the serialized length of the KeyValue key for a row: the 2-byte
row-length prefix, the
+ * row, and the 10-byte metadata suffix (column-family length, timestamp,
key type).
+ *
+ * <p>The data block and the root index block both write the full KeyValue
key (not just the row)
+ * so that a reader can parse and point-look-up either block: a point lookup
compares index keys
+ * against data keys, so the two must use byte-identical key encoding.
+ *
+ * @param rowLength length of the row (key content) in bytes.
+ * @return the KeyValue key length.
+ */
+ protected static int keyValueKeyLength(int rowLength) {
+ return SIZEOF_INT16 + rowLength + KEY_METADATA_SUFFIX_LENGTH;
+ }
+
+ /**
+ * Writes the KeyValue key for a row:
+ * {@code [2-byte rowLen][row][1-byte cfLen=0][8-byte ts=LATEST][1-byte
type=Put]}. See
+ * {@link #keyValueKeyLength(int)} for why the data and index blocks share
this encoding.
+ *
+ * @param out output stream to write to.
+ * @param row buffer holding the row (key content) bytes.
+ * @param offset start of the row within {@code row}; a key may be a view
into a larger buffer.
+ * @param rowLength number of row bytes to write; passed explicitly because
a key's backing array
+ * may be larger than its content length.
+ */
+ protected static void writeKey(DataOutputStream out, byte[] row, int offset,
int rowLength)
+ throws IOException {
+ out.writeShort((short) rowLength);
+ out.write(row, offset, rowLength);
+ out.write(0); // column-family length
+ out.writeLong(LATEST_TIMESTAMP); // timestamp
+ out.write(KEY_TYPE_PUT); // key type
+ }
+
/**
* Returns the bytes of the variable length encoding for an integer.
* @param length an integer, normally representing a length.
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
index 551cfd2c961b..1da721d0dc56 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
@@ -28,9 +28,6 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.hudi.io.hfile.DataSize.SIZEOF_BYTE;
-import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT16;
-import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT64;
import static
org.apache.hudi.io.hfile.HFileReader.SEEK_TO_BEFORE_BLOCK_FIRST_KEY;
import static org.apache.hudi.io.hfile.HFileReader.SEEK_TO_FOUND;
import static org.apache.hudi.io.hfile.HFileReader.SEEK_TO_IN_RANGE;
@@ -40,18 +37,11 @@ import static org.apache.hudi.io.hfile.KeyValue.KEY_OFFSET;
* Represents a {@link HFileBlockType#DATA} block.
*/
public class HFileDataBlock extends HFileBlock {
- private static final int KEY_LENGTH_LENGTH = SIZEOF_INT16;
- private static final int COLUMN_FAMILY_LENGTH = SIZEOF_BYTE;
- private static final int VERSION_TIMESTAMP_LENGTH = SIZEOF_INT64;
- private static final int KEY_TYPE_LENGTH = SIZEOF_BYTE;
// Hudi does not use HFile MVCC timestamp version so the version
// is always 0, thus the byte length of the version is always 1.
// This assumption is also validated when parsing {@link HFileInfo},
// i.e., the maximum MVCC timestamp in a HFile must be 0.
private static final long ZERO_TS_VERSION_BYTE_LENGTH = 1;
- // Hudi does not set version timestamp for key value pairs,
- // so the latest timestamp is used.
- private static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
// End offset of content in the block, relative to the start of the block.
The key-values
// occupy exactly uncompressedSizeWithoutHeader bytes after the header; the
checksum trails
@@ -218,24 +208,11 @@ public class HFileDataBlock extends HFileBlock {
// Length of key + length of a short variable indicating length of key.
// Note that 10 extra bytes are required by hbase reader.
// That is: 1 byte for column family length, 8 bytes for timestamp, 1
bytes for key type.
- dataOutputStream.writeInt(
- kv.key.length + KEY_LENGTH_LENGTH + COLUMN_FAMILY_LENGTH +
VERSION_TIMESTAMP_LENGTH + KEY_TYPE_LENGTH);
+ dataOutputStream.writeInt(keyValueKeyLength(kv.key.length));
// Length of value.
dataOutputStream.writeInt(kv.value.length);
- // Key content length.
- dataOutputStream.writeShort((short)kv.key.length);
// Key.
- dataOutputStream.write(kv.key);
- // Column family length: constant 0.
- dataOutputStream.write(0);
- // Column qualifier: assume 0 bits.
- // Timestamp: using the latest.
- dataOutputStream.writeLong(LATEST_TIMESTAMP);
- // Key type: constant Put (4) in Hudi.
- // Minimum((byte) 0), Put((byte) 4), Delete((byte) 8),
- // DeleteFamilyVersion((byte) 10), DeleteColumn((byte) 12),
- // DeleteFamily((byte) 14), Maximum((byte) 255).
- dataOutputStream.write(4);
+ writeKey(dataOutputStream, kv.key, 0, kv.key.length);
// Value.
dataOutputStream.write(kv.value);
// MVCC.
diff --git
a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileRootIndexBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileRootIndexBlock.java
index 5b32a5df8925..56a06ed1ba32 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileRootIndexBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileRootIndexBlock.java
@@ -29,7 +29,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
-import static org.apache.hudi.io.hfile.DataSize.SIZEOF_INT16;
import static org.apache.hudi.io.util.IOUtils.copy;
import static org.apache.hudi.io.util.IOUtils.decodeVarLongSizeOnDisk;
import static org.apache.hudi.io.util.IOUtils.readInt;
@@ -109,15 +108,11 @@ public class HFileRootIndexBlock extends HFileIndexBlock {
outputStream.writeLong(entry.getOffset());
outputStream.writeInt(entry.getSize());
- // Key length + 2 (SIZEOF_INT16 for the 2-byte row key length prefix).
- // Use Hadoop WritableUtils VarInt encoding to match HBase's HFile
format.
- byte[] keyLength = writeVarInt(
- entry.getFirstKey().getLength() + SIZEOF_INT16);
- outputStream.write(keyLength);
- // Key length.
- outputStream.writeShort((short) entry.getFirstKey().getLength());
- // Key.
- outputStream.write(entry.getFirstKey().getBytes());
+ // Use Hadoop WritableUtils VarInt encoding to match HBase's HFile
format and the reader.
+ int kvKeyLength = keyValueKeyLength(entry.getFirstKey().getLength());
+ outputStream.write(writeVarInt(kvKeyLength));
+ Key firstKey = entry.getFirstKey();
+ writeKey(outputStream, firstKey.getBytes(), firstKey.getOffset(),
firstKey.getLength());
}
}
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileDataBlock.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileDataBlock.java
new file mode 100644
index 000000000000..c040bca9a97e
--- /dev/null
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileDataBlock.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.io.hfile;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Validates the exact on-disk bytes the data block writer emits for each
record. A data entry is a
+ * full HBase KeyValue: {@code [4-byte keyLen][4-byte valueLen][2-byte
rowLen][row][1-byte cfLen=0]
+ * [8-byte ts=LATEST][1-byte type=Put][value][1-byte MVCC=0]}. An HBase reader
relies on this exact
+ * framing, so the test asserts every field rather than a single opaque blob.
+ */
+class TestHFileDataBlock {
+ private static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
+ private static final byte KEY_TYPE_PUT = (byte) 4;
+ // Column-family length (1) + timestamp (8) + key type (1) + the 2-byte
row-length prefix.
+ private static final int KEY_SUFFIX_AND_PREFIX_LENGTH = 12;
+
+ @Test
+ void writesFullHBaseKeyValuePerRecord() throws IOException {
+ HFileDataBlock block =
+ HFileDataBlock.createDataBlockToWrite(HFileContext.builder().build(),
-1L);
+ // Two records of different key/value lengths to verify the framing
repeats correctly.
+ block.add(utf8("key1"), utf8("value1"));
+ block.add(utf8("k22"), utf8("v2"));
+
+ ByteBuffer buf = block.getUncompressedBlockDataToWrite();
+ assertDataEntry(buf, "key1", "value1");
+ assertDataEntry(buf, "k22", "v2");
+ assertEquals(0, buf.remaining(), "unexpected trailing bytes after the last
record");
+ }
+
+ private static void assertDataEntry(ByteBuffer buf, String key, String
value) {
+ int rowLength = utf8(key).length;
+ int valueLength = utf8(value).length;
+ assertEquals(rowLength + KEY_SUFFIX_AND_PREFIX_LENGTH, buf.getInt(), "key
length for " + key);
+ assertEquals(valueLength, buf.getInt(), "value length for " + key);
+ assertEquals((short) rowLength, buf.getShort(), "row length for " + key);
+ assertEquals(key, readString(buf, rowLength), "row for " + key);
+ assertEquals((byte) 0, buf.get(), "column-family length for " + key);
+ assertEquals(LATEST_TIMESTAMP, buf.getLong(), "timestamp for " + key);
+ assertEquals(KEY_TYPE_PUT, buf.get(), "key type for " + key);
+ assertEquals(value, readString(buf, valueLength), "value for " + key);
+ assertEquals((byte) 0, buf.get(), "MVCC version for " + key);
+ }
+
+ private static byte[] utf8(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static String readString(ByteBuffer buf, int length) {
+ byte[] bytes = new byte[length];
+ buf.get(bytes);
+ return new String(bytes, StandardCharsets.UTF_8);
+ }
+}
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReadCompatibility.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReadCompatibility.java
index 8ca7ee7e5eab..c190a62fded3 100644
---
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReadCompatibility.java
+++
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReadCompatibility.java
@@ -22,6 +22,7 @@ package org.apache.hudi.io.hfile;
import org.apache.hudi.io.ByteArraySeekableDataInputStream;
import org.apache.hudi.io.ByteBufferBackedInputStream;
import org.apache.hudi.io.SeekableDataInputStream;
+import org.apache.hudi.io.compress.CompressionCodec;
import org.apache.hudi.io.util.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -29,19 +30,20 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparatorImpl;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.EnumSource;
+import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
@@ -57,6 +59,11 @@ import static
org.apache.hudi.io.hfile.HFileInfo.KEY_VALUE_VERSION;
import static org.apache.hudi.io.util.FileIOUtils.readAsByteArray;
import static org.apache.hudi.io.util.IOUtils.readInt;
import static org.apache.hudi.io.util.IOUtils.toBytes;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
class TestHFileReadCompatibility {
// Test data - simple key-value pairs
@@ -68,6 +75,11 @@ class TestHFileReadCompatibility {
new TestRecord("row5", "value5")
);
+ // Small block size + many records => many data blocks => block-boundary
keys land in the root
+ // index, which is exactly where the HBase point-lookup parsing happened.
+ private static final int MULTI_BLOCK_RECORDS = 2000;
+ private static final int SMALL_BLOCK_SIZE = 512;
+
@ParameterizedTest
@CsvSource({
"/hfile/hudi-generated.hfile,/hfile/hbase-generated.hfile",
@@ -78,8 +90,8 @@ class TestHFileReadCompatibility {
org.apache.hadoop.hbase.io.hfile.HFile.Reader hbaseReader =
createHBaseHFileReaderFromResource(hbaseFilePath)) {
// Validate number of entries.
- Assertions.assertEquals(5, hudiReader.getNumKeyValueEntries());
- Assertions.assertEquals(5, hbaseReader.getEntries());
+ assertEquals(5, hudiReader.getNumKeyValueEntries());
+ assertEquals(5, hbaseReader.getEntries());
// Validate data block content.
hudiReader.seekTo();
HFileScanner scanner = hbaseReader.getScanner(true, true);
@@ -89,60 +101,60 @@ class TestHFileReadCompatibility {
org.apache.hudi.io.hfile.KeyValue keyValue =
hudiReader.getKeyValue().get();
Cell cell = scanner.getCell();
// Ensure Hudi record is correct.
- Assertions.assertEquals(TEST_RECORDS.get(i).key,
keyValue.getKey().getContentInString());
+ assertEquals(TEST_RECORDS.get(i).key,
keyValue.getKey().getContentInString());
byte[] value = Arrays.copyOfRange(
keyValue.getBytes(),
keyValue.getValueOffset(),
keyValue.getValueOffset() + keyValue.getValueLength());
- Assertions.assertArrayEquals(value,
TEST_RECORDS.get(i).value.getBytes());
+ assertArrayEquals(value, TEST_RECORDS.get(i).value.getBytes());
// Ensure Hbase record is correct.
byte[] key = Arrays.copyOfRange(
cell.getRowArray(),
cell.getRowOffset(),
cell.getRowOffset() + cell.getRowLength());
- Assertions.assertArrayEquals(TEST_RECORDS.get(i).key.getBytes(), key);
+ assertArrayEquals(TEST_RECORDS.get(i).key.getBytes(), key);
value = Arrays.copyOfRange(
cell.getValueArray(),
cell.getValueOffset(),
cell.getValueOffset() + cell.getValueLength());
- Assertions.assertArrayEquals(value,
TEST_RECORDS.get(i).value.getBytes());
+ assertArrayEquals(value, TEST_RECORDS.get(i).value.getBytes());
i++;
} while (hudiReader.next() && scanner.next());
// Compare some meta information.
// LAST KEY.
-
Assertions.assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.LAST_KEY.getBytes()));
-
Assertions.assertTrue(hudiReader.getMetaInfo(HFileInfo.LAST_KEY).isPresent());
+
assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.LAST_KEY.getBytes()));
+ assertTrue(hudiReader.getMetaInfo(HFileInfo.LAST_KEY).isPresent());
// The last key value returned from hbase contains the extra fields,
// e.g., column family, column qualifier, timestamp, key type, which is
10 more bytes.
// Therefore, the last key value from hudi should be the prefix since
hudi does not use these
// extra fields.
if (hudiReader.getMetaInfo(HFileInfo.LAST_KEY).get().length
<
hbaseReader.getHFileInfo().get(HFileInfo.LAST_KEY.getBytes()).length) {
- Assertions.assertTrue(isPrefix(
+ assertTrue(isPrefix(
hudiReader.getMetaInfo(HFileInfo.LAST_KEY).get(),
hbaseReader.getHFileInfo().get(HFileInfo.LAST_KEY.getBytes())));
} else {
- Assertions.assertTrue(isPrefix(
+ assertTrue(isPrefix(
hbaseReader.getHFileInfo().get(HFileInfo.LAST_KEY.getBytes()),
hudiReader.getMetaInfo(HFileInfo.LAST_KEY).get()));
}
// Average key length.
-
Assertions.assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.AVG_KEY_LEN.getBytes()));
-
Assertions.assertTrue(hudiReader.getMetaInfo(HFileInfo.AVG_KEY_LEN).isPresent());
+
assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.AVG_KEY_LEN.getBytes()));
+ assertTrue(hudiReader.getMetaInfo(HFileInfo.AVG_KEY_LEN).isPresent());
// Average value length.
-
Assertions.assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.AVG_VALUE_LEN.getBytes()));
-
Assertions.assertTrue(hudiReader.getMetaInfo(HFileInfo.AVG_VALUE_LEN).isPresent());
- Assertions.assertTrue(
+
assertTrue(hbaseReader.getHFileInfo().containsKey(HFileInfo.AVG_VALUE_LEN.getBytes()));
+ assertTrue(hudiReader.getMetaInfo(HFileInfo.AVG_VALUE_LEN).isPresent());
+ assertTrue(
hbaseReader.getHFileInfo().getAvgValueLen()
>=
readInt(hudiReader.getMetaInfo(HFileInfo.AVG_VALUE_LEN).get(), 0));
// MVCC.
-
Assertions.assertTrue(hbaseReader.getHFileInfo().shouldIncludeMemStoreTS());
+ assertTrue(hbaseReader.getHFileInfo().shouldIncludeMemStoreTS());
// Note that MemStoreTS is not set.
- Assertions.assertFalse(hbaseReader.getHFileInfo().isDecodeMemstoreTS());
-
Assertions.assertTrue(hudiReader.getMetaInfo(KEY_VALUE_VERSION).isPresent());
-
Assertions.assertTrue(hudiReader.getMetaInfo(HFileInfo.MAX_MVCC_TS_KEY).isPresent());
- Assertions.assertEquals(0L,
+ assertFalse(hbaseReader.getHFileInfo().isDecodeMemstoreTS());
+ assertTrue(hudiReader.getMetaInfo(KEY_VALUE_VERSION).isPresent());
+
assertTrue(hudiReader.getMetaInfo(HFileInfo.MAX_MVCC_TS_KEY).isPresent());
+ assertEquals(0L,
IOUtils.readLong(
hudiReader.getMetaInfo(HFileInfo.MAX_MVCC_TS_KEY).get(), 0));
}
@@ -160,13 +172,13 @@ class TestHFileReadCompatibility {
// Create HBase HFile.Reader from the temporary file
HFile.Reader reader = HFile.createReader(fs, new
Path(tempFile.toString()), conf);
byte[] keyValueVersion =
reader.getHFileInfo().get(KEY_VALUE_VERSION.getBytes());
- Assertions.assertEquals(1, IOUtils.readInt(keyValueVersion, 0));
+ assertEquals(1, IOUtils.readInt(keyValueVersion, 0));
// Values from trailer still works.
- Assertions.assertEquals(5, reader.getEntries());
+ assertEquals(5, reader.getEntries());
// Scanning the file succeeds.
HFileScanner scanner = reader.getScanner(true, true);
scanner.seekTo();
- Assertions.assertDoesNotThrow(() -> {
+ assertDoesNotThrow(() -> {
int i = 0;
do {
Cell cell = scanner.getCell();
@@ -174,12 +186,118 @@ class TestHFileReadCompatibility {
cell.getRowArray(),
cell.getRowOffset(),
cell.getRowOffset() + cell.getRowLength());
- Assertions.assertArrayEquals(TEST_RECORDS.get(i).key.getBytes(), key);
+ assertArrayEquals(TEST_RECORDS.get(i).key.getBytes(), key);
i++;
} while (scanner.next());
});
}
+ /**
+ * Validates the block-index key encoding in the HFile: an HBase reader
point-looks-up every key
+ * (including the block-boundary keys stored in the root index) in a
native-written multi-block
+ * file and gets an exact match with the correct value.
+ */
+ @ParameterizedTest
+ @EnumSource(value = CompressionCodec.class, names = {"NONE", "GZIP"})
+ void hbaseReaderPointLooksUpEveryKeyInNativeMultiBlockFile(CompressionCodec
codec)
+ throws IOException {
+ byte[] data = writeMultiBlockHudiHFile(MULTI_BLOCK_RECORDS,
SMALL_BLOCK_SIZE, codec);
+ try (HFile.Reader reader = createHBaseHFileReader(data)) {
+ int blocks = reader.getTrailer().getDataIndexCount();
+ assertTrue(blocks > 1, "expected a multi-block file; got " + blocks);
+ assertEquals(MULTI_BLOCK_RECORDS, reader.getEntries());
+ HFileScanner scanner = reader.getScanner(true, true);
+ for (int i = 0; i < MULTI_BLOCK_RECORDS; i++) {
+ KeyValue probe = new KeyValue(Bytes.toBytes(key(i)), null, null, null);
+ assertEquals(0, scanner.seekTo(probe), "expected exact match for " +
key(i));
+ Cell cell = scanner.getCell();
+ assertEquals(key(i),
+ Bytes.toString(cell.getRowArray(), cell.getRowOffset(),
cell.getRowLength()));
+ assertEquals(value(i),
+ Bytes.toString(cell.getValueArray(), cell.getValueOffset(),
cell.getValueLength()));
+ }
+ }
+ }
+
+ /** Validates the block-index key encoding parses as a {@code KeyValue} via
{@code midKey()}. */
+ @Test
+ void hbaseReaderMidKeyParsesNativeBlockIndexKey() throws IOException {
+ byte[] data = writeMultiBlockHudiHFile(MULTI_BLOCK_RECORDS,
SMALL_BLOCK_SIZE, CompressionCodec.NONE);
+ try (HFile.Reader reader = createHBaseHFileReader(data)) {
+ assertTrue(reader.midKey().isPresent());
+ }
+ }
+
+ /**
+ * Byte comparison under both NONE and GZIP: given identical records, the
native writer and the
+ * HBase writer produce cells that an HBase reader sees as byte-identical
(key bytes and value
+ * bytes), establishing that the native writer emits HBase-format cells.
+ */
+ @ParameterizedTest
+ @EnumSource(value = CompressionCodec.class, names = {"NONE", "GZIP"})
+ void
nativeAndHBaseWrittenCellsAreByteIdenticalUnderHBaseReader(CompressionCodec
codec)
+ throws IOException {
+ byte[] nativeData = writeMultiBlockHudiHFile(MULTI_BLOCK_RECORDS,
SMALL_BLOCK_SIZE, codec);
+ byte[] hbaseData =
+ writeMultiBlockHBaseHFile(MULTI_BLOCK_RECORDS, SMALL_BLOCK_SIZE,
hbaseAlgo(codec));
+ try (HFile.Reader nativeReader = createHBaseHFileReader(nativeData);
+ HFile.Reader hbaseReader = createHBaseHFileReader(hbaseData)) {
+ assertEquals(hbaseReader.getEntries(), nativeReader.getEntries());
+ HFileScanner ns = nativeReader.getScanner(true, true);
+ HFileScanner hs = hbaseReader.getScanner(true, true);
+ assertTrue(ns.seekTo());
+ assertTrue(hs.seekTo());
+ int compared = 0;
+ boolean nativeHasNext;
+ boolean hbaseHasNext;
+ do {
+ Cell nativeCell = ns.getCell();
+ Cell hbaseCell = hs.getCell();
+ assertArrayEquals(keyBytes(nativeCell), keyBytes(hbaseCell),
+ "KeyValue key bytes differ at record " + compared);
+ assertArrayEquals(valueBytes(nativeCell), valueBytes(hbaseCell),
+ "value bytes differ at record " + compared);
+ compared++;
+ nativeHasNext = ns.next();
+ hbaseHasNext = hs.next();
+ } while (nativeHasNext && hbaseHasNext);
+ assertEquals(MULTI_BLOCK_RECORDS, compared);
+ assertFalse(nativeHasNext, "native scanner had extra cells");
+ assertFalse(hbaseHasNext, "hbase scanner had extra cells");
+ }
+ }
+
+ /**
+ * Cross-reader equivalence: the same native-written multi-block file reads
identically through
+ * the native hudi-io reader and the HBase reader (same rows and values, in
order).
+ */
+ @Test
+ void nativeWrittenFileReadsIdenticallyByBothReaders() throws IOException {
+ byte[] data = writeMultiBlockHudiHFile(MULTI_BLOCK_RECORDS,
SMALL_BLOCK_SIZE, CompressionCodec.NONE);
+ try (HFileReader nativeReader = createHFileReader(data);
+ HFile.Reader hbaseReader = createHBaseHFileReader(data)) {
+ assertEquals(MULTI_BLOCK_RECORDS, hbaseReader.getEntries());
+ nativeReader.seekTo();
+ HFileScanner hbaseScanner = hbaseReader.getScanner(true, true);
+ assertTrue(hbaseScanner.seekTo());
+ for (int i = 0; i < MULTI_BLOCK_RECORDS; i++) {
+ org.apache.hudi.io.hfile.KeyValue nativeKv =
nativeReader.getKeyValue().get();
+ Cell hbaseCell = hbaseScanner.getCell();
+ assertEquals(key(i), nativeKv.getKey().getContentInString());
+ assertEquals(key(i),
+ Bytes.toString(hbaseCell.getRowArray(), hbaseCell.getRowOffset(),
hbaseCell.getRowLength()));
+ byte[] nativeValue = Arrays.copyOfRange(nativeKv.getBytes(),
nativeKv.getValueOffset(),
+ nativeKv.getValueOffset() + nativeKv.getValueLength());
+ assertArrayEquals(value(i).getBytes(StandardCharsets.UTF_8),
nativeValue);
+ assertArrayEquals(nativeValue, valueBytes(hbaseCell));
+ if (i < MULTI_BLOCK_RECORDS - 1) {
+ assertTrue(nativeReader.next());
+ assertTrue(hbaseScanner.next());
+ }
+ }
+ }
+ }
+
static boolean isPrefix(byte[] prefix, byte[] array) {
if (prefix.length > array.length) {
return false; // can't be prefix if longer
@@ -193,39 +311,25 @@ class TestHFileReadCompatibility {
}
static HFileReader createHFileReaderFromResource(String fileName) throws
IOException {
- // Read HFile data from resources
- byte[] hfileData = readHFileFromResources(fileName);
- // Convert to ByteBuffer
- ByteBuffer buffer = ByteBuffer.wrap(hfileData);
- // Create SeekableDataInputStream
+ return createHFileReader(readHFileFromResources(fileName));
+ }
+
+ static HFileReader createHFileReader(byte[] hfileData) {
SeekableDataInputStream inputStream = new ByteArraySeekableDataInputStream(
- new ByteBufferBackedInputStream(buffer)
- );
- // Create and return HFileReaderImpl
+ new ByteBufferBackedInputStream(ByteBuffer.wrap(hfileData)));
return new HFileReaderImpl(inputStream, hfileData.length);
}
static HFile.Reader createHBaseHFileReaderFromResource(String fileName)
throws IOException {
- // Read HFile data from resources
- byte[] hfileData = readHFileFromResources(fileName);
- // Create a temporary file to write the HFile data
+ return createHBaseHFileReader(readHFileFromResources(fileName));
+ }
+
+ static HFile.Reader createHBaseHFileReader(byte[] hfileData) throws
IOException {
Path tempFile = new Path(Files.createTempFile("hbase_hfile_",
".hfile").toString());
- try {
- // Write the byte array to temporary file
- Files.write(Paths.get(tempFile.toString()), hfileData);
- // Create Hadoop Configuration and FileSystem
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
- // Create HBase HFile.Reader from the temporary file
- HFile.Reader reader = HFile.createReader(fs, new
Path(tempFile.toString()), conf);
- // Note: The temporary file will be cleaned up when the reader is closed
- // or you can manually delete it after use
- return reader;
- } catch (IOException e) {
- // Clean up temp file if creation fails
- Files.deleteIfExists(Paths.get(tempFile.toString()));
- throw e;
- }
+ Files.write(Paths.get(tempFile.toString()), hfileData);
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ return HFile.createReader(fs, tempFile, conf);
}
private static byte[] readHFileFromResources(String filename) throws
IOException {
@@ -250,7 +354,7 @@ class TestHFileReadCompatibility {
FileSystem fs = FileSystem.get(conf);
// Create HFile context with appropriate settings
- HFileContext context = new HFileContextBuilder()
+ org.apache.hadoop.hbase.io.hfile.HFileContext context = new
HFileContextBuilder()
.withBlockSize(64 * 1024)
.withCompression(Compression.Algorithm.NONE)
.withCellComparator(CellComparatorImpl.COMPARATOR)
@@ -279,7 +383,7 @@ class TestHFileReadCompatibility {
}
private void writeHFileWithHudi(Path filePath, int keyValueVersion) throws
IOException {
- org.apache.hudi.io.hfile.HFileContext context =
org.apache.hudi.io.hfile.HFileContext.builder()
+ HFileContext context = HFileContext.builder()
.blockSize(64 * 1024)
.build();
try (DataOutputStream outputStream = new DataOutputStream(
@@ -297,6 +401,67 @@ class TestHFileReadCompatibility {
}
}
+ private static String key(int i) {
+ return String.format("key%06d", i);
+ }
+
+ private static String value(int i) {
+ return "value-" + i;
+ }
+
+ private static Compression.Algorithm hbaseAlgo(CompressionCodec codec) {
+ return codec == CompressionCodec.GZIP ? Compression.Algorithm.GZ :
Compression.Algorithm.NONE;
+ }
+
+ private static byte[] writeMultiBlockHudiHFile(int numRecords, int
blockSize, CompressionCodec codec)
+ throws IOException {
+ HFileContext context = HFileContext.builder()
+ .blockSize(blockSize).compressionCodec(codec).build();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (HFileWriter writer = new HFileWriterImpl(context, baos)) {
+ for (int i = 0; i < numRecords; i++) {
+ writer.append(key(i), value(i).getBytes(StandardCharsets.UTF_8));
+ }
+ }
+ return baos.toByteArray();
+ }
+
+ // Same cells the native writer emits: family length 0, timestamp = LATEST,
type = Put.
+ private static byte[] writeMultiBlockHBaseHFile(int numRecords, int
blockSize,
+ Compression.Algorithm algo)
throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ Path path = new Path(Files.createTempFile("hbase_write_",
".hfile").toString());
+ org.apache.hadoop.hbase.io.hfile.HFileContext context = new
HFileContextBuilder()
+ .withBlockSize(blockSize)
+ .withCompression(algo)
+ .withCellComparator(CellComparatorImpl.COMPARATOR)
+ .withIncludesMvcc(true)
+ .build();
+ try (HFile.Writer writer = HFile.getWriterFactory(conf, new
CacheConfig(conf))
+ .withPath(fs, path).withFileContext(context).create()) {
+ for (int i = 0; i < numRecords; i++) {
+ writer.append(new KeyValue(Bytes.toBytes(key(i)), new byte[0], new
byte[0],
+ HConstants.LATEST_TIMESTAMP,
value(i).getBytes(StandardCharsets.UTF_8)));
+ }
+ }
+ return Files.readAllBytes(Paths.get(path.toString()));
+ }
+
+ private static byte[] keyBytes(Cell c) {
+ KeyValue kv = new KeyValue(c.getRowArray(), c.getRowOffset(),
c.getRowLength(),
+ c.getFamilyArray(), c.getFamilyOffset(), c.getFamilyLength(),
+ c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength(),
+ c.getTimestamp(), KeyValue.Type.codeToType(c.getTypeByte()),
+ c.getValueArray(), c.getValueOffset(), c.getValueLength());
+ return Arrays.copyOfRange(kv.getKey(), 0, kv.getKeyLength());
+ }
+
+ private static byte[] valueBytes(Cell c) {
+ return Arrays.copyOfRange(c.getValueArray(), c.getValueOffset(),
+ c.getValueOffset() + c.getValueLength());
+ }
+
// Simple test record class
private static class TestRecord {
final String key;
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileRootIndexBlock.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileRootIndexBlock.java
new file mode 100644
index 000000000000..bf24801ea258
--- /dev/null
+++
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileRootIndexBlock.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.io.hfile;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Validates the exact on-disk bytes the root index block writer emits for
each entry. An entry is
+ * {@code [8-byte offset][4-byte size][varint keyLen][2-byte
rowLen][row][1-byte cfLen=0]
+ * [8-byte ts=LATEST][1-byte type=Put]}: the block-index "first key" is a full
HBase KeyValue key,
+ * byte-identical to the data block's key, so an HBase reader can
point-look-up the block index.
+ */
+class TestHFileRootIndexBlock {
+ private static final long LATEST_TIMESTAMP = Long.MAX_VALUE;
+ private static final byte KEY_TYPE_PUT = (byte) 4;
+ // Column-family length (1) + timestamp (8) + key type (1) + the 2-byte
row-length prefix.
+ private static final int KEY_SUFFIX_AND_PREFIX_LENGTH = 12;
+
+ @Test
+ void writesFullHBaseKeyValueKeyPerEntry() throws IOException {
+ HFileRootIndexBlock block =
+
HFileRootIndexBlock.createRootIndexBlockToWrite(HFileContext.builder().build());
+ // Two entries of different key lengths, offsets, and sizes.
+ block.add(utf8("key1"), 0L, 100);
+ block.add(utf8("key0002"), 100L, 250);
+
+ ByteBuffer buf = block.getUncompressedBlockDataToWrite();
+ assertIndexEntry(buf, "key1", 0L, 100);
+ assertIndexEntry(buf, "key0002", 100L, 250);
+ assertEquals(0, buf.remaining(), "unexpected trailing bytes after the last
entry");
+ }
+
+ private static void assertIndexEntry(ByteBuffer buf, String key, long
offset, int size) {
+ int rowLength = utf8(key).length;
+ assertEquals(offset, buf.getLong(), "offset for " + key);
+ assertEquals(size, buf.getInt(), "size for " + key);
+ // The key length is a Hadoop WritableUtils VarInt; it is a single byte
here because
+ // rowLength + 12 < 128 for these keys (multi-byte VarInt is covered by
long-key read tests).
+ assertEquals(rowLength + KEY_SUFFIX_AND_PREFIX_LENGTH, buf.get() & 0xff,
+ "varint key length for " + key);
+ assertEquals((short) rowLength, buf.getShort(), "row length for " + key);
+ assertEquals(key, readString(buf, rowLength), "row for " + key);
+ assertEquals((byte) 0, buf.get(), "column-family length for " + key);
+ assertEquals(LATEST_TIMESTAMP, buf.getLong(), "timestamp for " + key);
+ assertEquals(KEY_TYPE_PUT, buf.get(), "key type for " + key);
+ }
+
+ private static byte[] utf8(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static String readString(ByteBuffer buf, int length) {
+ byte[] bytes = new byte[length];
+ buf.get(bytes);
+ return new String(bytes, StandardCharsets.UTF_8);
+ }
+}
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
index a4488351c7e2..e1995684c88d 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
@@ -24,6 +24,14 @@ import org.apache.hudi.io.ByteBufferBackedInputStream;
import org.apache.hudi.io.SeekableDataInputStream;
import lombok.extern.slf4j.Slf4j;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.CellComparatorImpl;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
@@ -55,6 +63,16 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class TestHFileWriter {
private static final String TEST_FILE = "test.hfile";
private static final HFileContext CONTEXT = HFileContext.builder().build();
+ // Golden bytes (NONE compression, fixed input) that lock the on-disk
encoding of the data block
+ // and the root block-index block. Update intentionally ONLY after reviewing
HBase-reader
+ // compatibility, since any change here is a storage-format change.
+ private static final String GOLDEN_DATA_REGION_HEX =
+
"44415441424c4b2a000000610000005dffffffffffffffff00000040000000007e000000100000000600046b65793100"
+ +
"7fffffffffffffff0476616c75653100000000100000000600046b657932007fffffffffffffff0476616c7565320000"
+ +
"0000100000000600046b657933007fffffffffffffff0476616c7565330000000000";
+ private static final String GOLDEN_ROOT_INDEX_BLOCK_HEX =
+
"494458524f4f5432000000210000001dffffffffffffffff00000040000000003e000000000000000000000082100004"
+ + "6b657931007fffffffffffffff0400000000";
@AfterEach
public void tearDown() throws IOException {
@@ -232,6 +250,72 @@ class TestHFileWriter {
}
}
+ /**
+ * Format lock: with NONE compression and a fixed input the data block and
root block-index block
+ * are deterministic, so their raw bytes are asserted against a golden. The
same records written
+ * by the HBase HFile writer (NONE compression, NULL checksum, latest
timestamp, Put type) must
+ * produce the same two block byte regions, proving the native and HBase
writers agree on the
+ * on-disk encoding. Any change to the encoding (dropping the KeyValue
suffix, ts/type, or
+ * framing) fails here. Neither block holds the file-creation timestamp, so
the bytes are stable.
+ */
+ @Test
+ void writerBlockBytesAreStableFormatLock() throws Exception {
+ writeTestFile();
+ String[] nativeBlocks =
dataAndRootIndexBlockHex(Files.readAllBytes(Paths.get(TEST_FILE)));
+ // Logged so the golden can be regenerated intentionally.
+ log.info("GOLDEN_DATA_REGION_HEX={}", nativeBlocks[0]);
+ log.info("GOLDEN_ROOT_INDEX_BLOCK_HEX={}", nativeBlocks[1]);
+ assertEquals(GOLDEN_DATA_REGION_HEX, nativeBlocks[0],
+ "native data block bytes changed (storage-format change); review HBase
compatibility");
+ assertEquals(GOLDEN_ROOT_INDEX_BLOCK_HEX, nativeBlocks[1],
+ "native root block-index bytes changed (storage-format change); review
HBase compatibility");
+
+ // The HBase writer, given the same records, must produce the same two
block byte regions.
+ String[] hbaseBlocks = dataAndRootIndexBlockHex(writeFixedHBaseFile());
+ log.info("HBASE_DATA_REGION_HEX={}", hbaseBlocks[0]);
+ log.info("HBASE_ROOT_INDEX_BLOCK_HEX={}", hbaseBlocks[1]);
+ assertEquals(GOLDEN_DATA_REGION_HEX, hbaseBlocks[0],
+ "HBase writer data block bytes differ from the native writer");
+ assertEquals(GOLDEN_ROOT_INDEX_BLOCK_HEX, hbaseBlocks[1],
+ "HBase writer root block-index bytes differ from the native writer");
+ }
+
+ /** Returns {@code [dataRegionHex, rootIndexBlockHex]} for an HFile's raw
bytes. */
+ private static String[] dataAndRootIndexBlockHex(byte[] data) {
+ int idxRootOffset = indexOf(data, HFileBlockType.ROOT_INDEX.getMagic());
+ assertTrue(idxRootOffset > 0, "root index block not found");
+ // Root index block: 33-byte v3 block header + onDiskSizeWithoutHeader
payload (at header + 8).
+ int onDiskSizeWithoutHeader = readIntBE(data, idxRootOffset + 8);
+ return new String[] {
+ hex(Arrays.copyOfRange(data, 0, idxRootOffset)),
+ hex(Arrays.copyOfRange(data, idxRootOffset, idxRootOffset + 33 +
onDiskSizeWithoutHeader))
+ };
+ }
+
+ /** Writes key1/key2/key3 with the HBase HFile writer, matching the native
writer's settings. */
+ private static byte[] writeFixedHBaseFile() throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ org.apache.hadoop.fs.Path path =
+ new org.apache.hadoop.fs.Path(Files.createTempFile("hbase_fixed_",
".hfile").toString());
+ org.apache.hadoop.hbase.io.hfile.HFileContext context = new
HFileContextBuilder()
+ .withBlockSize(1024 * 1024)
+ .withCompression(Compression.Algorithm.NONE)
+ .withChecksumType(org.apache.hadoop.hbase.util.ChecksumType.NULL)
+ .withCellComparator(CellComparatorImpl.COMPARATOR)
+ .withIncludesMvcc(true)
+ .build();
+ try (HFile.Writer writer = HFile.getWriterFactory(conf, new
CacheConfig(conf))
+ .withPath(fs, path).withFileContext(context).create()) {
+ for (int i = 1; i <= 3; i++) {
+ writer.append(new org.apache.hadoop.hbase.KeyValue(
+ ("key" + i).getBytes(StandardCharsets.UTF_8), new byte[0], new
byte[0],
+ HConstants.LATEST_TIMESTAMP, ("value" +
i).getBytes(StandardCharsets.UTF_8)));
+ }
+ }
+ return Files.readAllBytes(Paths.get(path.toString()));
+ }
+
private static void writeTestFile() throws Exception {
try (
DataOutputStream outputStream =
@@ -246,7 +330,9 @@ class TestHFileWriter {
private static void validateHFileSize() throws IOException {
Path path = Paths.get(TEST_FILE);
long actualSize = Files.size(path);
- long expectedSize = 4537;
+ // Each root block-index entry carries the 10-byte HBase KeyValue suffix
(column-family
+ // length + timestamp + key type). This file has one index entry, so the
size grows by 10.
+ long expectedSize = 4547;
assertEquals(expectedSize, actualSize);
}
@@ -339,6 +425,32 @@ class TestHFileWriter {
}
}
+ private static int indexOf(byte[] haystack, byte[] needle) {
+ outer:
+ for (int i = 0; i + needle.length <= haystack.length; i++) {
+ for (int j = 0; j < needle.length; j++) {
+ if (haystack[i + j] != needle[j]) {
+ continue outer;
+ }
+ }
+ return i;
+ }
+ return -1;
+ }
+
+ private static int readIntBE(byte[] b, int off) {
+ return ((b[off] & 0xff) << 24) | ((b[off + 1] & 0xff) << 16)
+ | ((b[off + 2] & 0xff) << 8) | (b[off + 3] & 0xff);
+ }
+
+ private static String hex(byte[] b) {
+ StringBuilder sb = new StringBuilder(b.length * 2);
+ for (byte x : b) {
+ sb.append(Character.forDigit((x >> 4) & 0xf,
16)).append(Character.forDigit(x & 0xf, 16));
+ }
+ return sb.toString();
+ }
+
public static String generateRandomStringStream(int length) {
String characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
Random random = new Random();
