This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a27828d8de75 test(io): drive the HFile writer tests through byte
arrays instead of a file (#19618)
a27828d8de75 is described below
commit a27828d8de756d2c871f7f5d9d255e73c261aa1d
Author: Ranga Reddy <[email protected]>
AuthorDate: Wed Aug 26 15:32:09 2026 +0530
test(io): drive the HFile writer tests through byte arrays instead of a
file (#19618)
* test(io): drive the HFile writer tests through byte arrays instead of a
file
TestHFileWriter wrote every fixture to "test.hfile", a fixed relative path
in the module
working directory, then read it back through FileChannel.map.
HFileWriterImpl already
accepts any OutputStream and HFileReaderImpl already reads from a
ByteArraySeekableDataInputStream - three sibling tests in this package
build readers that
way - so the file was never needed.
Writes now go to a ByteArrayOutputStream and reads go through a new
openReader(byte[])
helper, matching the idiom already used by TestHFileMultiBlockScan,
TestHFileCompatibility and TestHFileReadCompatibility. No production code
changes.
What this removes beyond the file itself:
- the fixed relative path shared by four test methods, which was only
safe because they
never run concurrently;
- a hidden coupling where assertSingleBlockBytesMatchGoldenAndHBase
re-read the file that
assertSingleBlockBytesMatchHBase had left behind. The latter now
returns the bytes and
the caller passes them on;
- the @AfterEach that existed solely to delete the file, and the
FileChannel /
StandardOpenOption imports;
- three leaked readers: the old code closed the FileChannel but not the
HFileReaderImpl
opened inside it. Every reader now sits in try-with-resources.
Renamed for what the helpers actually do now that no path is involved:
writeNativeFile ->
writeNativeHFile (returns the bytes), writeTestFile ->
writeThreeRecordHFile, and the
validate* helpers take the bytes rather than reading a file behind the
caller's back.
The HBase writer fixture keeps its temp file: that writer needs a Hadoop
Path, and it is
created with Files.createTempFile rather than a fixed name.
hudi-io: 125 tests pass, checkstyle and apache-rat clean.
Closes #17413
* test(io): finish the byte-array sweep in TestHFileWriter
- Drive the HBase fixture through an in-memory FSDataOutputStream via
HFile.WriterFactory.withOutputStream. The writer never needed a Path,
and the createTempFile it used leaked four files per run.
- Drop the DataOutputStream wrapper around ByteArrayOutputStream at the
three native write sites; HFileWriterImpl takes a plain OutputStream
and closes it itself. The catch-and-rethrow-as-RuntimeException blocks
went with it since both tests already declare IOException.
- Route the remaining String.getBytes() calls through bytes() so the
whole file encodes as UTF-8.
- Reuse CONTEXT and hoist the shared three-record fixture; narrow
writeThreeRecordHFile to throws IOException.
---------
Co-authored-by: voon <[email protected]>
---
.../org/apache/hudi/io/hfile/TestHFileWriter.java | 179 ++++++++-------------
1 file changed, 70 insertions(+), 109 deletions(-)
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
index 23e8d7007961..e33cf65389fe 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileWriter.java
@@ -21,13 +21,12 @@ package org.apache.hudi.io.hfile;
import org.apache.hudi.io.ByteArraySeekableDataInputStream;
import org.apache.hudi.io.ByteBufferBackedInputStream;
-import org.apache.hudi.io.SeekableDataInputStream;
import org.apache.hudi.io.hfile.protobuf.generated.HFileProtos;
import org.apache.hudi.io.util.IOUtils;
import com.google.protobuf.CodedOutputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.hbase.CellComparatorImpl;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.io.compress.Compression;
@@ -36,22 +35,15 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.WritableUtils;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
@@ -78,8 +70,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class TestHFileWriter {
private static final Logger LOG =
LoggerFactory.getLogger(TestHFileWriter.class);
- private static final String TEST_FILE = "test.hfile";
private static final HFileContext CONTEXT = HFileContext.builder().build();
+ private static final String[] THREE_KEYS = {"key1", "key2", "key3"};
+ private static final byte[][] THREE_VALUES = {bytes("value1"),
bytes("value2"), bytes("value3")};
// Golden bytes (NONE compression, fixed input) that lock the on-disk
encoding of the data block
// and the root block-index block. Update intentionally ONLY after reviewing
HBase-reader
// compatibility, since any change here is a storage-format change.
@@ -105,21 +98,16 @@ class TestHFileWriter {
+
"616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161"
+
"616161616161616161616161616161616161616161007fffffffffffffff0400000000";
- @AfterEach
- public void tearDown() throws IOException {
- Files.deleteIfExists(Paths.get(TEST_FILE));
- }
-
@Test
void testOverflow() throws Exception {
// 1. Write data.
- writeTestFile();
+ byte[] hfile = writeThreeRecordHFile();
// 2. Validate file size.
- validateHFileSize();
+ validateHFileSize(hfile);
// 3. Validate file structure.
- validateHFileStructure();
+ validateHFileStructure(hfile);
// 4. Validate consistency with HFileReader.
- validateConsistencyWithHFileReader();
+ validateConsistencyWithHFileReader(hfile);
LOG.info("All validations passed!");
}
@@ -127,39 +115,29 @@ class TestHFileWriter {
void testSameKeyLocation() throws IOException {
// 165 bytes for data part limit.
HFileContext context = new HFileContext.Builder().blockSize(165).build();
- String testFile = TEST_FILE;
+ ByteArrayOutputStream hfileBytes = new ByteArrayOutputStream();
// CREATE 4 BLOCKs:
// Block 1: 100 records, whose keys are the same: "key00".
// Block 2: 5 records, whose first key is "key01"
// Block 3: 5 records, whose first key is "key06"
// Block 4: 1 record, whose first key is "key11",
// whose length is larger than the block size.
- try (DataOutputStream outputStream =
- new DataOutputStream(Files.newOutputStream(Paths.get(testFile)));
- HFileWriter writer = new HFileWriterImpl(context, outputStream)) {
+ try (HFileWriter writer = new HFileWriterImpl(context, hfileBytes)) {
// All entries for key00 are stored in the first block.
for (int i = 0; i < 100; i++) {
- writer.append("key00", String.format("value%02d", i).getBytes());
+ writer.append("key00", bytes(String.format("value%02d", i)));
}
// Otherwise, 5 records in each other blocks.
for (int i = 1; i < 11; i++) {
- writer.append(
- String.format("key%02d", i),
- String.format("value%02d", i).getBytes());
+ writer.append(String.format("key%02d", i),
bytes(String.format("value%02d", i)));
}
// Adding a record whose size is larger than block size.
String longValue = generateRandomStringStream(200);
- writer.append("key11", longValue.getBytes());
- } catch (IOException e) {
- throw new RuntimeException(e);
+ writer.append("key11", bytes(longValue));
}
// Validate.
- try (FileChannel channel = FileChannel.open(Paths.get(testFile),
StandardOpenOption.READ)) {
- ByteBuffer buf = channel.map(FileChannel.MapMode.READ_ONLY, 0,
channel.size());
- SeekableDataInputStream inputStream =
- new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(buf));
- HFileReaderImpl reader = new HFileReaderImpl(inputStream,
channel.size());
+ try (HFileReaderImpl reader = openReader(hfileBytes.toByteArray())) {
reader.initializeMetadata();
// Totally 111 records.
assertEquals(111, reader.getNumKeyValueEntries());
@@ -194,24 +172,16 @@ class TestHFileWriter {
void testUniqueKeyLocation() throws IOException {
// 50 bytes for data part limit.
HFileContext context = new HFileContext.Builder().blockSize(100).build();
- String testFile = TEST_FILE;
- try (DataOutputStream outputStream =
- new DataOutputStream(Files.newOutputStream(Paths.get(testFile)));
- HFileWriter writer = new HFileWriterImpl(context, outputStream)) {
+ ByteArrayOutputStream hfileBytes = new ByteArrayOutputStream();
+ try (HFileWriter writer = new HFileWriterImpl(context, hfileBytes)) {
for (int i = 0; i < 50; i++) {
- writer.append(
- String.format("key%02d", i), String.format("value%02d",
i).getBytes());
+ writer.append(String.format("key%02d", i),
bytes(String.format("value%02d", i)));
}
- } catch (IOException e) {
- throw new RuntimeException(e);
}
// Validate.
- try (FileChannel channel = FileChannel.open(Paths.get(testFile),
StandardOpenOption.READ)) {
- ByteBuffer buf = channel.map(FileChannel.MapMode.READ_ONLY, 0,
channel.size());
- SeekableDataInputStream inputStream =
- new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(buf));
- HFileReaderImpl reader = new HFileReaderImpl(inputStream,
channel.size());
+ byte[] hfile = hfileBytes.toByteArray();
+ try (HFileReaderImpl reader = openReader(hfile)) {
reader.initializeMetadata();
assertEquals(50, reader.getNumKeyValueEntries());
HFileTrailer trailer = reader.getTrailer();
@@ -220,10 +190,10 @@ class TestHFileWriter {
for (int i = 0; i < 50; i++) {
KeyValue kv = reader.getKeyValue().get();
assertArrayEquals(
- String.format("key%02d", i).getBytes(),
- kv.getKey().getContentInString().getBytes());
+ bytes(String.format("key%02d", i)),
+ bytes(kv.getKey().getContentInString()));
assertArrayEquals(
- String.format("value%02d", i).getBytes(),
+ bytes(String.format("value%02d", i)),
Arrays.copyOfRange(
kv.getBytes(),
kv.getValueOffset(),
@@ -235,12 +205,11 @@ class TestHFileWriter {
// Each data block's previous-block-offset header (8 bytes at offset 16)
must chain back to
// the prior block, and -1 for the first, so an HBase reader's
seekBefore can step back across
// blocks. The writer once set it to the block's own offset; this guards
that regression.
- byte[] file = Files.readAllBytes(Paths.get(testFile));
List<IndexEntry> dataIndex = parseIndexBlock(
- file, (int) trailer.getLoadOnOpenDataOffset(),
trailer.getDataIndexCount());
+ hfile, (int) trailer.getLoadOnOpenDataOffset(),
trailer.getDataIndexCount());
assertTrue(dataIndex.size() > 1, "test setup: expected multiple data
blocks");
for (int i = 0; i < dataIndex.size(); i++) {
- long prevOffset = readLongBE(file, (int) dataIndex.get(i).offset + 16);
+ long prevOffset = readLongBE(hfile, (int) dataIndex.get(i).offset +
16);
long expected = i == 0 ? -1L : dataIndex.get(i - 1).offset;
assertEquals(expected, prevOffset, "data block " + i + "
previous-block offset");
}
@@ -261,9 +230,7 @@ class TestHFileWriter {
void writerBlockBytesAreStableFormatLock() throws Exception {
byte[][] vals = {bytes("v0"), bytes("v1"), bytes("v2")};
assertSingleBlockBytesMatchGoldenAndHBase(
- new String[] {"key1", "key2", "key3"},
- new byte[][] {bytes("value1"), bytes("value2"), bytes("value3")},
- GOLDEN_DATA_REGION_HEX, GOLDEN_ROOT_INDEX_BLOCK_HEX);
+ THREE_KEYS, THREE_VALUES, GOLDEN_DATA_REGION_HEX,
GOLDEN_ROOT_INDEX_BLOCK_HEX);
assertSingleBlockBytesMatchGoldenAndHBase(
new String[] {rep('a', 116), rep('b', 8), rep('c', 12)}, vals,
GOLDEN_VARLEN_DATA_REGION_HEX, GOLDEN_VARLEN_ROOT_INDEX_BLOCK_HEX);
@@ -316,8 +283,7 @@ class TestHFileWriter {
*/
private static void assertSingleBlockBytesMatchGoldenAndHBase(
String[] keys, byte[][] values, String dataGolden, String rootGolden)
throws Exception {
- assertSingleBlockBytesMatchHBase(keys, values, dataGolden, rootGolden);
- byte[] nativeFile = Files.readAllBytes(Paths.get(TEST_FILE));
+ byte[] nativeFile = assertSingleBlockBytesMatchHBase(keys, values,
dataGolden, rootGolden);
HFileProtos.TrailerProto trailer = validateTrailerSection(nativeFile,
keys.length, 0);
int loadOnOpen = (int) trailer.getLoadOnOpenDataOffset();
int metaIndexOffset = loadOnOpen + HFILEBLOCK_HEADER_SIZE +
readIntBE(nativeFile, loadOnOpen + 8);
@@ -331,11 +297,12 @@ class TestHFileWriter {
* Asserts the native and HBase writers produce identical single-block data
and root index bytes
* (optionally also matching a pinned golden), and the root-index keyLength
is Hadoop VInt. Used
* without a golden for large keys, where a hardcoded golden of multi-KB key
bytes adds no signal.
+ *
+ * @return the native writer's HFile bytes, so callers can assert further on
the same file.
*/
- private static void assertSingleBlockBytesMatchHBase(
+ private static byte[] assertSingleBlockBytesMatchHBase(
String[] keys, byte[][] values, String dataGolden, String rootGolden)
throws Exception {
- writeNativeFile(TEST_FILE, HFileContext.builder().build(), keys, values,
null, null);
- byte[] nativeFile = Files.readAllBytes(Paths.get(TEST_FILE));
+ byte[] nativeFile = writeNativeHFile(CONTEXT, keys, values, null, null);
String[] nativeBlocks = dataAndRootIndexBlockHex(nativeFile);
String[] hbaseBlocks = dataAndRootIndexBlockHex(writeHBaseFile(keys,
values));
assertEquals(hbaseBlocks[0], nativeBlocks[0], "native vs HBase data block
bytes");
@@ -349,6 +316,7 @@ class TestHFileWriter {
IndexEntry first =
parseIndexBlock(nativeFile, indexOf(nativeFile,
ROOT_INDEX.getMagic()), 1).get(0);
assertVarIntIsHadoopNotProtobuf("root index keyLength", first);
+ return nativeFile;
}
/** Returns {@code [dataRegionHex, rootIndexBlockHex]} for an HFile's raw
bytes. */
@@ -363,12 +331,14 @@ class TestHFileWriter {
};
}
- /** Writes the given records with the HBase HFile writer, matching the
native writer's settings. */
+ /**
+ * Writes the given records with the HBase HFile writer, matching the native
writer's settings,
+ * returning the complete file as bytes. The HBase writer only needs a
position-tracking stream,
+ * so it is driven through an in-memory {@link FSDataOutputStream} rather
than a temp file.
+ */
private static byte[] writeHBaseFile(String[] keys, byte[][] values) throws
IOException {
Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf);
- org.apache.hadoop.fs.Path path =
- new org.apache.hadoop.fs.Path(Files.createTempFile("hbase_fixed_",
".hfile").toString());
+ ByteArrayOutputStream hfileBytes = new ByteArrayOutputStream();
org.apache.hadoop.hbase.io.hfile.HFileContext context = new
HFileContextBuilder()
.withBlockSize(1024 * 1024)
.withCompression(Compression.Algorithm.NONE)
@@ -376,15 +346,16 @@ class TestHFileWriter {
.withCellComparator(CellComparatorImpl.COMPARATOR)
.withIncludesMvcc(true)
.build();
- try (HFile.Writer writer = HFile.getWriterFactory(conf, new
CacheConfig(conf))
- .withPath(fs, path).withFileContext(context).create()) {
+ try (FSDataOutputStream outputStream = new FSDataOutputStream(hfileBytes,
null);
+ HFile.Writer writer = HFile.getWriterFactory(conf, new
CacheConfig(conf))
+
.withOutputStream(outputStream).withFileContext(context).create()) {
for (int i = 0; i < keys.length; i++) {
writer.append(new org.apache.hadoop.hbase.KeyValue(
keys[i].getBytes(StandardCharsets.UTF_8), new byte[0], new byte[0],
HConstants.LATEST_TIMESTAMP, values[i]));
}
}
- return Files.readAllBytes(Paths.get(path.toString()));
+ return hfileBytes.toByteArray();
}
/**
@@ -405,9 +376,8 @@ class TestHFileWriter {
String metaKey = makeKey(200, 9999);
byte[] metaValue = bytes("bloom-filter-payload-bytes");
// Small block size forces one record per block, so the root index has
multiple entries.
- writeNativeFile(
- TEST_FILE, new HFileContext.Builder().blockSize(100).build(), keys,
values, metaKey, metaValue);
- byte[] file = Files.readAllBytes(Paths.get(TEST_FILE));
+ byte[] file = writeNativeHFile(
+ new HFileContext.Builder().blockSize(100).build(), keys, values,
metaKey, metaValue);
HFileProtos.TrailerProto trailer = validateTrailerSection(file,
numRecords, 1);
int loadOnOpen = (int) trailer.getLoadOnOpenDataOffset();
@@ -415,7 +385,7 @@ class TestHFileWriter {
int metaIndexOffset = loadOnOpen + HFILEBLOCK_HEADER_SIZE +
readIntBE(file, loadOnOpen + 8);
validateMetaIndexAndBlock(file, metaIndexOffset, metaKey, metaValue);
validateFileInfoSection(file, metaIndexOffset, (int)
trailer.getFileInfoOffset(), keys[numRecords - 1]);
- assertAllRecordsReadBack(numRecords, keys);
+ assertAllRecordsReadBack(file, numRecords, keys);
}
/** Validates the trailer magic, protobuf framing, fields, and HFile
version; returns the proto. */
@@ -503,11 +473,9 @@ class TestHFileWriter {
}
/** Reads every record back through the native reader. */
- private static void assertAllRecordsReadBack(int numRecords, String[] keys)
throws IOException {
- try (FileChannel channel = FileChannel.open(Paths.get(TEST_FILE),
StandardOpenOption.READ)) {
- ByteBuffer buf = channel.map(FileChannel.MapMode.READ_ONLY, 0,
channel.size());
- HFileReaderImpl reader = new HFileReaderImpl(
- new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(buf)), channel.size());
+ private static void assertAllRecordsReadBack(byte[] hfile, int numRecords,
String[] keys)
+ throws IOException {
+ try (HFileReaderImpl reader = openReader(hfile)) {
reader.initializeMetadata();
assertEquals(numRecords, reader.getNumKeyValueEntries());
reader.seekTo();
@@ -518,13 +486,15 @@ class TestHFileWriter {
}
}
- /** Writes the given records (and an optional single meta block) with the
native HFile writer. */
- private static void writeNativeFile(String testFile, HFileContext context,
String[] keys,
- byte[][] values, String metaKey, byte[]
metaValue)
+ /**
+ * Writes the given records (and an optional single meta block) with the
native HFile writer,
+ * returning the complete file as bytes.
+ */
+ private static byte[] writeNativeHFile(HFileContext context, String[] keys,
+ byte[][] values, String metaKey,
byte[] metaValue)
throws IOException {
- try (DataOutputStream outputStream =
- new DataOutputStream(Files.newOutputStream(Paths.get(testFile)));
- HFileWriter writer = new HFileWriterImpl(context, outputStream)) {
+ ByteArrayOutputStream hfileBytes = new ByteArrayOutputStream();
+ try (HFileWriter writer = new HFileWriterImpl(context, hfileBytes)) {
for (int i = 0; i < keys.length; i++) {
writer.append(keys[i], values[i]);
}
@@ -532,6 +502,13 @@ class TestHFileWriter {
writer.appendMetaInfo(metaKey, metaValue);
}
}
+ return hfileBytes.toByteArray();
+ }
+
+ /** Opens the native reader over an in-memory HFile. */
+ private static HFileReaderImpl openReader(byte[] hfile) {
+ return new HFileReaderImpl(
+ new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(hfile)), hfile.length);
}
/** A parsed block index entry: block offset, on-disk size, raw keyLength
vint bytes, and key. */
@@ -642,28 +619,21 @@ class TestHFileWriter {
return v;
}
- private static void writeTestFile() throws Exception {
- try (
- DataOutputStream outputStream =
- new DataOutputStream(Files.newOutputStream(Paths.get(TEST_FILE)));
- HFileWriter writer = new HFileWriterImpl(CONTEXT, outputStream)) {
- writer.append("key1", "value1".getBytes());
- writer.append("key2", "value2".getBytes());
- writer.append("key3", "value3".getBytes());
- }
+ /** Writes a three-record HFile with the default context, returning the file
bytes. */
+ private static byte[] writeThreeRecordHFile() throws IOException {
+ return writeNativeHFile(CONTEXT, THREE_KEYS, THREE_VALUES, null, null);
}
- private static void validateHFileSize() throws IOException {
- Path path = Paths.get(TEST_FILE);
- long actualSize = Files.size(path);
+ private static void validateHFileSize(byte[] hfile) {
+ long actualSize = hfile.length;
// Each root block-index entry carries the 10-byte HBase KeyValue suffix
(column-family
// length + timestamp + key type). This file has one index entry, so the
size grows by 10.
long expectedSize = 4547;
assertEquals(expectedSize, actualSize);
}
- private static void validateHFileStructure() throws IOException {
- ByteBuffer fileBuffer = mapFileToBuffer();
+ private static void validateHFileStructure(byte[] hfile) {
+ ByteBuffer fileBuffer = ByteBuffer.wrap(hfile);
// 1. Validate Trailer
validateTrailer(fileBuffer);
@@ -672,11 +642,8 @@ class TestHFileWriter {
validateDataBlocks(fileBuffer);
}
- private static void validateConsistencyWithHFileReader() throws IOException {
- ByteBuffer content = mapFileToBuffer();
- try (HFileReader reader = new HFileReaderImpl(
- new ByteArraySeekableDataInputStream(
- new ByteBufferBackedInputStream(content)), content.limit())) {
+ private static void validateConsistencyWithHFileReader(byte[] hfile) throws
IOException {
+ try (HFileReader reader = openReader(hfile)) {
reader.initializeMetadata();
assertEquals(3, reader.getNumKeyValueEntries());
assertTrue(reader.getMetaInfo(LAST_KEY).isPresent());
@@ -688,12 +655,6 @@ class TestHFileWriter {
}
}
- private static ByteBuffer mapFileToBuffer() throws IOException {
- try (FileChannel channel = FileChannel.open(Paths.get(TEST_FILE),
StandardOpenOption.READ)) {
- return channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
- }
- }
-
private static void validateTrailer(ByteBuffer buf) {
int trailerStart = Math.max(0, buf.limit() - 4096);
buf.position(trailerStart);
