This is an automated email from the ASF dual-hosted git repository.
prashantwason pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new fe1c31d385bd fix(hfile): use long cursor offsets in native HFile
reader for files larger than 2GB (#20309)
fe1c31d385bd is described below
commit fe1c31d385bd5cd8ec57ea06757bbc461a1c03ae
Author: Prashant Wason <[email protected]>
AuthorDate: Tue Oct 13 21:51:28 2026 -0700
fix(hfile): use long cursor offsets in native HFile reader for files larger
than 2GB (#20309)
HFileCursor stored the absolute position in the file as an int, and
HFileReaderImpl cast block offsets to int. For HFiles larger than 2GB the
cursor offset overflowed, so isAtFirstKeyOfBlock() never matched and a valid
lookup in [fake first key, actual first key) after
SEEK_TO_BEFORE_BLOCK_FIRST_KEY
was rejected as a backward seekTo with IllegalStateException.
- Make HFileCursor offsets long and pass long block start offsets to
HFileDataBlock.seekTo/next; offsets relative to a block stay int.
- Remove the int casts on block offsets in HFileReaderImpl.
- Position the cursor in seekTo() using the first data block's offset
instead of the first key's offset within its byte array.
- Add a test that runs the point and prefix lookups with data block offsets
shifted beyond 2GB and 4GB.
Closes #20308
---
.../java/org/apache/hudi/io/hfile/HFileCursor.java | 7 +-
.../org/apache/hudi/io/hfile/HFileDataBlock.java | 8 +-
.../org/apache/hudi/io/hfile/HFileReaderImpl.java | 14 ++--
.../org/apache/hudi/io/hfile/TestHFileReader.java | 90 ++++++++++++++++++++++
4 files changed, 105 insertions(+), 14 deletions(-)
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
index 3aeacdbed239..d629e7464ac7 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileCursor.java
@@ -34,7 +34,8 @@ public class HFileCursor {
private static final int INVALID_POSITION = -1;
- private int offset;
+ // Absolute position in the HFile, which can exceed Integer.MAX_VALUE for
files larger than 2GB
+ private long offset;
private Option<KeyValue> keyValue;
@Getter(AccessLevel.NONE)
private boolean eof;
@@ -53,12 +54,12 @@ public class HFileCursor {
return !(offset == INVALID_POSITION || eof);
}
- public void set(int offset, KeyValue keyValue) {
+ public void set(long offset, KeyValue keyValue) {
this.offset = offset;
this.keyValue = Option.of(keyValue);
}
- public void setOffset(int offset) {
+ public void setOffset(long offset) {
this.offset = offset;
this.keyValue = Option.empty();
}
diff --git a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
index 1da721d0dc56..d655f6fafdd5 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileDataBlock.java
@@ -93,8 +93,8 @@ public class HFileDataBlock extends HFileBlock {
* of the data block; the cursor points to the actual first key of the data
block which is
* lexicographically greater than the lookup key.
*/
- int seekTo(HFileCursor cursor, Key key, int blockStartOffsetInFile) {
- int relativeOffset = cursor.getOffset() - blockStartOffsetInFile;
+ int seekTo(HFileCursor cursor, Key key, long blockStartOffsetInFile) {
+ int relativeOffset = (int) (cursor.getOffset() - blockStartOffsetInFile);
int lastRelativeOffset = relativeOffset;
Option<KeyValue> lastKeyValue = cursor.getKeyValue();
while (relativeOffset < uncompressedContentEndRelativeOffset) {
@@ -158,8 +158,8 @@ public class HFileDataBlock extends HFileBlock {
* HFile.
* @return {@code true} if there is next {@link KeyValue}; {code false}
otherwise.
*/
- boolean next(HFileCursor cursor, int blockStartOffsetInFile) {
- int offset = cursor.getOffset() - blockStartOffsetInFile;
+ boolean next(HFileCursor cursor, long blockStartOffsetInFile) {
+ int offset = (int) (cursor.getOffset() - blockStartOffsetInFile);
Option<KeyValue> keyValue = cursor.getKeyValue();
if (!keyValue.isPresent()) {
keyValue = Option.of(readKeyValue(offset));
diff --git
a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReaderImpl.java
b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReaderImpl.java
index bb04272a2883..04fca56559b3 100644
--- a/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReaderImpl.java
+++ b/hudi-io/src/main/java/org/apache/hudi/io/hfile/HFileReaderImpl.java
@@ -149,7 +149,7 @@ public class HFileReaderImpl implements HFileReader {
currentDataBlockEntry = Option.of(floorEntry.getValue());
currentDataBlock =
Option.of(instantiateHFileDataBlock(currentDataBlockEntry.get()));
cursor.setOffset(
- (int) currentDataBlockEntry.get().getOffset() +
HFILEBLOCK_HEADER_SIZE);
+ currentDataBlockEntry.get().getOffset() +
HFILEBLOCK_HEADER_SIZE);
}
}
if (!currentDataBlockEntry.get().getNextBlockFirstKey().isPresent()) {
@@ -170,7 +170,7 @@ public class HFileReaderImpl implements HFileReader {
}
return currentDataBlock.get()
- .seekTo(cursor, key, (int) currentDataBlockEntry.get().getOffset());
+ .seekTo(cursor, key, currentDataBlockEntry.get().getOffset());
}
if (compareCurrent == 0) {
return SEEK_TO_FOUND;
@@ -207,9 +207,9 @@ public class HFileReaderImpl implements HFileReader {
return false;
}
// Move the current position to the beginning of the first data block
- cursor.setOffset(dataBlockIndexEntryMap.firstKey().getOffset() +
HFILEBLOCK_HEADER_SIZE);
- cursor.unsetEof();
currentDataBlockEntry =
Option.of(dataBlockIndexEntryMap.firstEntry().getValue());
+ cursor.setOffset(currentDataBlockEntry.get().getOffset() +
HFILEBLOCK_HEADER_SIZE);
+ cursor.unsetEof();
// The data block will be read when {@link #getKeyValue} is called
currentDataBlock = Option.empty();
return true;
@@ -221,7 +221,7 @@ public class HFileReaderImpl implements HFileReader {
if (!currentDataBlock.isPresent()) {
currentDataBlock =
Option.of(instantiateHFileDataBlock(currentDataBlockEntry.get()));
}
- if (currentDataBlock.get().next(cursor, (int)
currentDataBlockEntry.get().getOffset())) {
+ if (currentDataBlock.get().next(cursor,
currentDataBlockEntry.get().getOffset())) {
// The position is advanced by the data block instance
return true;
}
@@ -231,7 +231,7 @@ public class HFileReaderImpl implements HFileReader {
cursor.setEof();
return false;
}
- cursor.setOffset((int) currentDataBlockEntry.get().getOffset() +
HFILEBLOCK_HEADER_SIZE);
+ cursor.setOffset(currentDataBlockEntry.get().getOffset() +
HFILEBLOCK_HEADER_SIZE);
return true;
}
return false;
@@ -247,7 +247,7 @@ public class HFileReaderImpl implements HFileReader {
}
keyValue =
Option.of(currentDataBlock.get().readKeyValue(
- cursor.getOffset() - (int)
currentDataBlockEntry.get().getOffset()));
+ (int) (cursor.getOffset() -
currentDataBlockEntry.get().getOffset())));
cursor.setKeyValue(keyValue.get());
}
return keyValue;
diff --git
a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
index 255502c569f4..d2b084a2cda2 100644
--- a/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
+++ b/hudi-io/src/test/java/org/apache/hudi/io/hfile/TestHFileReader.java
@@ -22,6 +22,7 @@ package org.apache.hudi.io.hfile;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.io.ByteArraySeekableDataInputStream;
import org.apache.hudi.io.ByteBufferBackedInputStream;
+import org.apache.hudi.io.SeekableDataInputStream;
import lombok.AllArgsConstructor;
import lombok.Getter;
@@ -36,6 +37,7 @@ import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
+import java.util.TreeMap;
import java.util.function.Function;
import java.util.stream.Stream;
@@ -71,6 +73,11 @@ public class TestHFileReader {
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-";
private static final Function<Integer, String> LARGE_KEY_CREATOR = i ->
LARGE_KEY_PREFIX + String.format("%09d", i);
private static final int SEEK_TO_THROW_EXCEPTION = -3;
+ // Shifts applied to the data block offsets to simulate HFiles larger than
2GB: the first one
+ // makes the data blocks straddle the 2GB boundary, the others place all
data blocks beyond
+ // 2GB and 4GB respectively
+ private static final long[] DATA_BLOCK_OFFSET_SHIFTS = {
+ Integer.MAX_VALUE - 64 * 1024L, 3L << 30, 5L << 30};
static Stream<Arguments> testArgsReadHFilePointAndPrefixLookup() {
return Stream.of(
@@ -683,6 +690,31 @@ public class TestHFileReader {
verifyHFileRead(filename, numEntries, keyCreator, VALUE_CREATOR,
keyLookUpInfoList);
}
+ static Stream<Arguments> testArgsReadHFilePointAndPrefixLookupBeyond2GB() {
+ return testArgsReadHFilePointAndPrefixLookup().flatMap(arguments ->
+ Arrays.stream(DATA_BLOCK_OFFSET_SHIFTS).mapToObj(shift -> {
+ Object[] args = arguments.get();
+ Object[] argsWithShift = Arrays.copyOf(args, args.length + 1);
+ argsWithShift[args.length] = shift;
+ return Arguments.of(argsWithShift);
+ }));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testArgsReadHFilePointAndPrefixLookupBeyond2GB")
+ public void testReadHFilePointAndPrefixLookupBeyond2GB(String filename,
+ int numEntries,
+ Function<Integer,
String> keyCreator,
+ List<KeyLookUpInfo>
keyLookUpInfoList,
+ long
dataBlockOffsetShift) throws IOException {
+ byte[] content = readHFileFromResources(filename);
+ try (HFileReader reader = new DataBlockOffsetShiftingHFileReader(content,
dataBlockOffsetShift)) {
+ reader.initializeMetadata();
+ verifyHFileValuesInSequentialReads(reader, numEntries,
Option.of(keyCreator), Option.of(VALUE_CREATOR));
+ verifyHFileSeekToReads(reader, keyLookUpInfoList);
+ }
+ }
+
@Test
public void testReadHFileWithNonUniqueKeys() throws IOException {
try (HFileReader reader =
getHFileReader("/hfile/hudi_1_0_hbase_2_4_9_16KB_GZ_200_20_non_unique.hfile")) {
@@ -993,6 +1025,64 @@ public class TestHFileReader {
}
}
+ /**
+ * An {@link HFileReaderImpl} that reads the data blocks of an HFile as if
they were stored at
+ * offsets shifted by {@code dataBlockOffsetShift}, to simulate reading an
HFile larger than 2GB
+ * without materializing such a file.
+ */
+ private static class DataBlockOffsetShiftingHFileReader extends
HFileReaderImpl {
+ private final long dataBlockOffsetShift;
+
+ DataBlockOffsetShiftingHFileReader(byte[] content, long
dataBlockOffsetShift) {
+ super(new OffsetShiftingSeekableDataInputStream(content,
dataBlockOffsetShift), content.length);
+ this.dataBlockOffsetShift = dataBlockOffsetShift;
+ }
+
+ @Override
+ public synchronized void initializeMetadata() throws IOException {
+ if (isMetadataInitialized) {
+ return;
+ }
+ super.initializeMetadata();
+ TreeMap<Key, BlockIndexEntry> shiftedDataBlockIndexEntryMap = new
TreeMap<>();
+ dataBlockIndexEntryMap.forEach((key, entry) ->
shiftedDataBlockIndexEntryMap.put(key,
+ new BlockIndexEntry(entry.getFirstKey(),
entry.getNextBlockFirstKey(),
+ entry.getOffset() + dataBlockOffsetShift, entry.getSize())));
+ dataBlockIndexEntryMap = shiftedDataBlockIndexEntryMap;
+ }
+ }
+
+ /**
+ * A {@link SeekableDataInputStream} that maps positions at or beyond {@code
shift} back to
+ * {@code position - shift} in the underlying content.
+ */
+ private static class OffsetShiftingSeekableDataInputStream extends
SeekableDataInputStream {
+ private final SeekableDataInputStream delegate;
+ private final long shift;
+ private long appliedShift = 0;
+
+ OffsetShiftingSeekableDataInputStream(byte[] content, long shift) {
+ this(new ByteArraySeekableDataInputStream(new
ByteBufferBackedInputStream(content)), shift);
+ }
+
+ private OffsetShiftingSeekableDataInputStream(SeekableDataInputStream
delegate, long shift) {
+ super(delegate);
+ this.delegate = delegate;
+ this.shift = shift;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return delegate.getPos() + appliedShift;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ appliedShift = pos >= shift ? shift : 0;
+ delegate.seek(pos - appliedShift);
+ }
+ }
+
@AllArgsConstructor
@Getter
static class KeyLookUpInfo {
