This is an automated email from the ASF dual-hosted git repository. RyanSkraba pushed a commit to branch branch-1.12 in repository https://gitbox.apache.org/repos/asf/avro.git
commit 14a0097c5a1b40e06b31a13166d911cc4d727781 Author: Ismaël Mejía <[email protected]> AuthorDate: Thu Aug 6 17:52:46 2026 +0200 AVRO-4323: [Java] Bound DataFileStream block size against available input before allocating the block buffer (#3919) * AVRO-4323: [Java] Bound DataFileStream block size against available input When reading a data (container) file, DataFileStream validated the declared block size only against the Integer range before allocating the block buffer. For a malformed, corrupted, or truncated file the declared size can greatly exceed the bytes actually present, so the reader eagerly allocated a large buffer before reading any block byte. Reject a declared block size that exceeds the number of bytes remaining in the input when that count is known (byte-array- or known-length-stream-backed decoders), so reading a malformed file fails fast with a clear IOException. The check is skipped when the remaining count is unknown (-1). * AVRO-4323: Address review: account for sync marker in block-size guard; close reader in test Include the trailing sync-marker length (DataFileConstants.SYNC_SIZE) in the remaining-bytes check so a file truncated right before the sync marker is rejected before allocating the block buffer, and use try-with-resources in the regression test. --- .../java/org/apache/avro/file/DataFileStream.java | 14 +++++ .../java/org/apache/avro/TestDataFileReader.java | 60 ++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java index 9b55c88b08..23d9edaff1 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java +++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileStream.java @@ -320,6 +320,20 @@ public class DataFileStream<D> implements Iterator<D>, Iterable<D>, Closeable { if (blockSize > Integer.MAX_VALUE || blockSize < 0) { throw new IOException("Block size invalid or too large for this implementation: " + blockSize); } + // When the number of bytes remaining in the input is known (e.g. a + // byte-array- or known-length-stream-backed decoder), reject a declared + // block size that could not possibly be satisfied by the data available. + // Reading a block consumes the block bytes plus the trailing sync marker, + // so both must fit in the remaining input. This avoids eagerly allocating + // a large block buffer (see the DataBlock constructor) for a malformed, + // corrupted, or truncated file before any block byte has been read. A + // value of -1 means the remaining count is unknown, in which case the + // check is skipped. + int remaining = vin.remainingBytes(); + if (remaining >= 0 && blockSize > (long) remaining - DataFileConstants.SYNC_SIZE) { + throw new IOException("Block size " + blockSize + " plus sync marker exceeds the number of bytes remaining in " + + "the input (" + remaining + "). The file is likely corrupted or truncated."); + } blockCount = blockRemaining; availableBlock = true; return true; diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java b/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java index 0e13c8388c..a0f3fa5562 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestDataFileReader.java @@ -319,4 +319,64 @@ public class TestDataFileReader { return output.toByteArray(); } + + /** + * A block header may declare a block size much larger than the data actually + * present in a corrupted or truncated file. When the remaining byte count is + * known, the reader must reject such a block up front rather than attempting to + * allocate a buffer of the declared size. + */ + @Test + void oversizedBlockSizeIsRejectedBeforeAllocation() throws IOException { + Schema schema = new Schema.Parser().parse("{\"type\":\"int\"}"); + + // A spec-correct header with zero records, so no real data block is written. + ByteArrayOutputStream fileBytes = new ByteArrayOutputStream(); + try (DataFileWriter<Object> w = new DataFileWriter<>(new GenericDatumWriter<>(schema))) { + w.create(schema, fileBytes); + } + + // Append a single block header that declares a huge block size but no data. + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(fileBytes, null); + encoder.writeLong(1L); // block entry count + encoder.writeLong(2_000_000_000L); // block size in bytes, far larger than what follows + encoder.flush(); + + byte[] malformed = fileBytes.toByteArray(); + + // hasNextBlock() surfaces block-header IOExceptions wrapped in an + // AvroRuntimeException, the same way the existing "block size too large" + // check does. + AvroRuntimeException exception = assertThrows(AvroRuntimeException.class, () -> { + try (DataFileStream<Object> reader = new DataFileStream<>(new ByteArrayInputStream(malformed), + new GenericDatumReader<>())) { + while (reader.hasNext()) { + reader.next(); + } + } + }); + assertNotNull(exception.getMessage()); + assertTrue(exception.getMessage().contains("Block size"), "Unexpected message: " + exception.getMessage()); + } + + /** + * A valid single-record file must still read normally after the guard is added. + */ + @Test + void validFileWithSingleRecordStillReads() throws IOException { + Schema schema = new Schema.Parser().parse("{\"type\":\"int\"}"); + + ByteArrayOutputStream fileBytes = new ByteArrayOutputStream(); + try (DataFileWriter<Object> w = new DataFileWriter<>(new GenericDatumWriter<>(schema))) { + w.create(schema, fileBytes); + w.append(42); + } + + try (DataFileStream<Object> reader = new DataFileStream<>(new ByteArrayInputStream(fileBytes.toByteArray()), + new GenericDatumReader<>())) { + assertTrue(reader.hasNext()); + assertEquals(42, reader.next()); + assertFalse(reader.hasNext()); + } + } }
