This is an automated email from the ASF dual-hosted git repository.
RyanSkraba pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/main by this push:
new 23aad5216f AVRO-4323: [Java] Bound DataFileStream block size against
available input before allocating the block buffer (#3919)
23aad5216f is described below
commit 23aad5216f874d0b7f3f2f43826cfb336747a970
Author: Ismaël Mejía <[email protected]>
AuthorDate: Thu Aug 6 17:52:46 2026 +0200
AVRO-4323: [Java] Bound DataFileStream block size against available input
before allocating the block buffer (#3919)
* AVRO-4323: [Java] Bound DataFileStream block size against available input
When reading a data (container) file, DataFileStream validated the declared
block size only against the Integer range before allocating the block
buffer.
For a malformed, corrupted, or truncated file the declared size can greatly
exceed the bytes actually present, so the reader eagerly allocated a large
buffer before reading any block byte.
Reject a declared block size that exceeds the number of bytes remaining in
the
input when that count is known (byte-array- or known-length-stream-backed
decoders), so reading a malformed file fails fast with a clear IOException.
The check is skipped when the remaining count is unknown (-1).
* AVRO-4323: Address review: account for sync marker in block-size guard;
close reader in test
Include the trailing sync-marker length (DataFileConstants.SYNC_SIZE) in the
remaining-bytes check so a file truncated right before the sync marker is
rejected before allocating the block buffer, and use try-with-resources in
the
regression test.
---
.../java/org/apache/avro/file/DataFileStream.java | 14 +++++
.../java/org/apache/avro/TestDataFileReader.java | 60 ++++++++++++++++++++++
2 files changed, 74 insertions(+)
diff --git
a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
index 9bb15183ec..fe95c76a15 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java
@@ -320,6 +320,20 @@ public class DataFileStream<D> implements Iterator<D>,
Iterable<D>, Closeable {
if (blockSize > Integer.MAX_VALUE || blockSize < 0) {
throw new IOException("Block size invalid or too large for this
implementation: " + blockSize);
}
+ // When the number of bytes remaining in the input is known (e.g. a
+ // byte-array- or known-length-stream-backed decoder), reject a declared
+ // block size that could not possibly be satisfied by the data available.
+ // Reading a block consumes the block bytes plus the trailing sync
marker,
+ // so both must fit in the remaining input. This avoids eagerly
allocating
+ // a large block buffer (see the DataBlock constructor) for a malformed,
+ // corrupted, or truncated file before any block byte has been read. A
+ // value of -1 means the remaining count is unknown, in which case the
+ // check is skipped.
+ int remaining = vin.remainingBytes();
+ if (remaining >= 0 && blockSize > (long) remaining -
DataFileConstants.SYNC_SIZE) {
+ throw new IOException("Block size " + blockSize + " plus sync marker
exceeds the number of bytes remaining in "
+ + "the input (" + remaining + "). The file is likely corrupted or
truncated.");
+ }
blockCount = blockRemaining;
availableBlock = true;
return true;
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java
b/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java
index 0e13c8388c..a0f3fa5562 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java
@@ -319,4 +319,64 @@ public class TestDataFileReader {
return output.toByteArray();
}
+
+ /**
+ * A block header may declare a block size much larger than the data actually
+ * present in a corrupted or truncated file. When the remaining byte count is
+ * known, the reader must reject such a block up front rather than
attempting to
+ * allocate a buffer of the declared size.
+ */
+ @Test
+ void oversizedBlockSizeIsRejectedBeforeAllocation() throws IOException {
+ Schema schema = new Schema.Parser().parse("{\"type\":\"int\"}");
+
+ // A spec-correct header with zero records, so no real data block is
written.
+ ByteArrayOutputStream fileBytes = new ByteArrayOutputStream();
+ try (DataFileWriter<Object> w = new DataFileWriter<>(new
GenericDatumWriter<>(schema))) {
+ w.create(schema, fileBytes);
+ }
+
+ // Append a single block header that declares a huge block size but no
data.
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(fileBytes,
null);
+ encoder.writeLong(1L); // block entry count
+ encoder.writeLong(2_000_000_000L); // block size in bytes, far larger than
what follows
+ encoder.flush();
+
+ byte[] malformed = fileBytes.toByteArray();
+
+ // hasNextBlock() surfaces block-header IOExceptions wrapped in an
+ // AvroRuntimeException, the same way the existing "block size too large"
+ // check does.
+ AvroRuntimeException exception = assertThrows(AvroRuntimeException.class,
() -> {
+ try (DataFileStream<Object> reader = new DataFileStream<>(new
ByteArrayInputStream(malformed),
+ new GenericDatumReader<>())) {
+ while (reader.hasNext()) {
+ reader.next();
+ }
+ }
+ });
+ assertNotNull(exception.getMessage());
+ assertTrue(exception.getMessage().contains("Block size"), "Unexpected
message: " + exception.getMessage());
+ }
+
+ /**
+ * A valid single-record file must still read normally after the guard is
added.
+ */
+ @Test
+ void validFileWithSingleRecordStillReads() throws IOException {
+ Schema schema = new Schema.Parser().parse("{\"type\":\"int\"}");
+
+ ByteArrayOutputStream fileBytes = new ByteArrayOutputStream();
+ try (DataFileWriter<Object> w = new DataFileWriter<>(new
GenericDatumWriter<>(schema))) {
+ w.create(schema, fileBytes);
+ w.append(42);
+ }
+
+ try (DataFileStream<Object> reader = new DataFileStream<>(new
ByteArrayInputStream(fileBytes.toByteArray()),
+ new GenericDatumReader<>())) {
+ assertTrue(reader.hasNext());
+ assertEquals(42, reader.next());
+ assertFalse(reader.hasNext());
+ }
+ }
}
