This is an automated email from the ASF dual-hosted git repository. iemejia pushed a commit to branch fuzzer in repository https://gitbox.apache.org/repos/asf/avro.git
commit bcde6c68f25acc61a35b34d065bbe36a7f2fd0b1 Author: Ismaël Mejía <[email protected]> AuthorDate: Sat Apr 4 16:39:03 2026 +0000 Tighten Avro fuzz coverage and input consumption checks Require successful decodes to consume full inputs, exercise direct decoding with short reads, add container reader-schema resolution targets, and expand the regression corpus to cover the new paths. --- lang/java/fuzz/REPORT.md | 24 ++++++++++- .../org/apache/avro/fuzz/BinaryDecodingFuzzer.java | 6 ++- .../org/apache/avro/fuzz/DataFileReaderFuzzer.java | 32 +++++++++++++++ .../java/org/apache/avro/fuzz/FuzzSupport.java | 44 ++++++++++++++++++++- .../org/apache/avro/fuzz/JsonDecodingFuzzer.java | 8 +++- .../org/apache/avro/fuzz/SeedCorpusGenerator.java | 33 ++++++++++++++++ .../seed-valid-record-single-node.bin | Bin 0 -> 25 bytes .../seed-valid-record-single-node.bin | Bin 0 -> 25 bytes .../fuzzDataFileReader/seed-valid-container.avro | Bin 516 -> 516 bytes .../seed-valid-container.avro | Bin 516 -> 516 bytes .../fuzzDataFileStream/seed-valid-container.avro | Bin 516 -> 516 bytes .../seed-valid-container.avro | Bin 516 -> 516 bytes .../seed-valid-record-null-extra.json | 1 + .../fuzzSchemaParse/seed-valid-alias-default.avsc | 1 + 14 files changed, 143 insertions(+), 6 deletions(-) diff --git a/lang/java/fuzz/REPORT.md b/lang/java/fuzz/REPORT.md index 115a3df849..36a70a5ca6 100644 --- a/lang/java/fuzz/REPORT.md +++ b/lang/java/fuzz/REPORT.md @@ -31,6 +31,25 @@ The module is wired into `lang/java/pom.xml` and can be run in regression or con - **Impact**: tiny hostile inputs could trigger huge allocations before EOF was detected - **Status**: Fixed +### Bug 3: `NullPointerException` for missing container schema metadata + +- **Severity**: Medium +- **Area**: `DataFileStream.initialize()` / `DataFileReader` header parsing +- **Impact**: malformed object container files missing `avro.schema` metadata could throw `NullPointerException` instead of a normal `IOException` +- **Status**: Fixed + +## Review Findings Addressed + +The module hardening work also addressed the following review findings: + +- fuzz targets were catching broad `Exception`, which hid genuine findings from Jazzer +- `fuzzDataFileStream` had no dedicated regression corpus directory +- the checked-in corpus mostly contained crash reproducers and little successful-path coverage +- schema fuzzing targeted deprecated `Schema.parse(...)` APIs rather than `SchemaParser` +- binary fuzzing only covered the buffered decoder path, not `directBinaryDecoder(...)` +- the fuzz module owned a hard-coded JUnit Platform launcher version instead of inheriting shared versioning from the Java parent +- the original report and run guidance had drifted from the actual module behavior + ## Current Module Improvements The fuzz module has been strengthened beyond the initial implementation: @@ -42,7 +61,10 @@ The fuzz module has been strengthened beyond the initial implementation: - binary decoding uses writer/reader schema pairs to exercise aliases, defaults, recursion, fixed, and logical types - regression corpora now include valid seeds in addition to historical crash reproducers - container-file corpora now cover both `fuzzDataFileReader` and `fuzzDataFileStream` -- module dependency alignment now follows `${junit5.version}` from the parent POM +- module dependency alignment now follows shared `${junit-platform.version}` from the Java parent POM +- successful binary and JSON decodes now require full input consumption +- the direct binary target now uses a short-read stream to exercise hostile streaming behavior +- container-file fuzzing now also covers reader-schema resolution paths ## BinaryDecoder Fix Notes diff --git a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/BinaryDecodingFuzzer.java b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/BinaryDecodingFuzzer.java index 665a6f3aa0..4912cddfb5 100644 --- a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/BinaryDecodingFuzzer.java +++ b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/BinaryDecodingFuzzer.java @@ -23,7 +23,6 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.DecoderFactory; -import java.io.ByteArrayInputStream; import java.io.IOException; /** @@ -50,11 +49,14 @@ class BinaryDecodingFuzzer { private void fuzz(byte[] data, boolean direct) { try { - BinaryDecoder decoder = direct ? DecoderFactory.get().directBinaryDecoder(new ByteArrayInputStream(data), null) + BinaryDecoder decoder = direct ? DecoderFactory.get().directBinaryDecoder(FuzzSupport.shortReadStream(data), null) : DecoderFactory.get().binaryDecoder(data, null); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(FuzzSupport.BINARY_WRITER_SCHEMA, FuzzSupport.BINARY_READER_SCHEMA); reader.read(null, decoder); + if (!direct && !decoder.isEnd()) { + throw new AssertionError("Buffered binary decoder left trailing bytes unread"); + } } catch (IOException e) { // Expected for malformed binary data } catch (RuntimeException e) { diff --git a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/DataFileReaderFuzzer.java b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/DataFileReaderFuzzer.java index 8c84acc942..481f545168 100644 --- a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/DataFileReaderFuzzer.java +++ b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/DataFileReaderFuzzer.java @@ -60,6 +60,22 @@ class DataFileReaderFuzzer { } } + @FuzzTest + void fuzzDataFileReaderWithResolution(byte[] data) { + try (DataFileReader<GenericRecord> reader = new DataFileReader<>(new SeekableByteArrayInput(data), + new GenericDatumReader<>(null, FuzzSupport.BINARY_READER_SCHEMA))) { + while (reader.hasNext()) { + reader.next(); + } + } catch (IOException e) { + // Expected for malformed container files + } catch (RuntimeException e) { + if (!FuzzSupport.isExpectedDecodingFailure(e)) { + throw e; + } + } + } + /** * Fuzz the streaming {@link DataFileStream} path via * {@link ByteArrayInputStream}, exercising sequential (non-seekable) container @@ -80,4 +96,20 @@ class DataFileReaderFuzzer { } } } + + @FuzzTest + void fuzzDataFileStreamWithResolution(byte[] data) { + try (DataFileStream<GenericRecord> stream = new DataFileStream<>(new ByteArrayInputStream(data), + new GenericDatumReader<>(null, FuzzSupport.BINARY_READER_SCHEMA))) { + while (stream.hasNext()) { + stream.next(); + } + } catch (IOException e) { + // Expected for malformed container files + } catch (RuntimeException e) { + if (!FuzzSupport.isExpectedDecodingFailure(e)) { + throw e; + } + } + } } diff --git a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/FuzzSupport.java b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/FuzzSupport.java index 98d93c1e4b..e62e33d76f 100644 --- a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/FuzzSupport.java +++ b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/FuzzSupport.java @@ -25,6 +25,11 @@ import org.apache.avro.SchemaParseException; import org.apache.avro.SystemLimitException; import org.apache.avro.UnresolvedUnionException; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + final class FuzzSupport { static final Schema BINARY_WRITER_SCHEMA = new Schema.Parser() .parse("{\"type\":\"record\",\"name\":\"WriterRoot\",\"fields\":[" + "{\"name\":\"id\",\"type\":\"long\"}," @@ -67,8 +72,20 @@ final class FuzzSupport { static boolean isExpectedDecodingFailure(RuntimeException exception) { return exception instanceof AvroTypeException || exception instanceof SystemLimitException - || exception instanceof UnresolvedUnionException || exception instanceof IllegalArgumentException - || exception instanceof UnsupportedOperationException || isExpectedAvroRuntimeFailure(exception); + || exception instanceof UnresolvedUnionException || isExpectedDecodingIllegalArgument(exception) + || isExpectedUnsupportedOperation(exception) || isExpectedAvroRuntimeFailure(exception); + } + + static InputStream shortReadStream(byte[] data) { + return new FilterInputStream(new ByteArrayInputStream(data)) { + @Override + public int read(byte[] buffer, int off, int len) throws IOException { + if (len <= 0) { + return super.read(buffer, off, len); + } + return super.read(buffer, off, Math.min(len, 3)); + } + }; } static String buildSchemaInput(FuzzedDataProvider data) { @@ -138,6 +155,29 @@ final class FuzzSupport { || message.startsWith("Not an array") || message.startsWith("Not a map") || message.startsWith("No match for "); } + private static boolean isExpectedDecodingIllegalArgument(RuntimeException exception) { + if (!(exception instanceof IllegalArgumentException)) { + return false; + } + + String message = exception.getMessage(); + if (message == null) { + return false; + } + + return message.contains("Invalid UTF-8") || message.contains("fromIndex") || message.contains("toIndex") + || message.contains("length") || message.contains("bound"); + } + + private static boolean isExpectedUnsupportedOperation(RuntimeException exception) { + if (!(exception instanceof UnsupportedOperationException)) { + return false; + } + + String message = exception.getMessage(); + return message != null && message.startsWith("Cannot read "); + } + private static String escapeJson(String value) { StringBuilder escaped = new StringBuilder(value.length()); for (int i = 0; i < value.length(); i++) { diff --git a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/JsonDecodingFuzzer.java b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/JsonDecodingFuzzer.java index 3c08705191..f51946aabe 100644 --- a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/JsonDecodingFuzzer.java +++ b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/JsonDecodingFuzzer.java @@ -24,7 +24,9 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.JsonDecoder; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; /** * Fuzz tests for Avro JSON decoding. @@ -39,10 +41,14 @@ class JsonDecodingFuzzer { @FuzzTest void fuzzJsonDecoding(FuzzedDataProvider data) { String jsonData = FuzzSupport.buildJsonInput(data); + ByteArrayInputStream input = new ByteArrayInputStream(jsonData.getBytes(StandardCharsets.UTF_8)); try { - JsonDecoder decoder = DecoderFactory.get().jsonDecoder(FuzzSupport.JSON_SCHEMA, jsonData); + JsonDecoder decoder = DecoderFactory.get().jsonDecoder(FuzzSupport.JSON_SCHEMA, input); GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(FuzzSupport.JSON_SCHEMA); reader.read(null, decoder); + if (input.available() > 0) { + throw new AssertionError("JSON decoder left trailing bytes unread"); + } } catch (IOException e) { // Expected for malformed JSON input } catch (RuntimeException e) { diff --git a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/SeedCorpusGenerator.java b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/SeedCorpusGenerator.java index f5d4ee4237..4252317ea4 100644 --- a/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/SeedCorpusGenerator.java +++ b/lang/java/fuzz/src/test/java/org/apache/avro/fuzz/SeedCorpusGenerator.java @@ -38,9 +38,17 @@ class SeedCorpusGenerator { @Test void generateSeeds() throws IOException { writeBytes("BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/seed-valid-record.bin", writeBinaryRecord()); + writeBytes("BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/seed-valid-record-single-node.bin", + writeBinaryRecordWithSingleNode()); writeBytes("BinaryDecodingFuzzerInputs/fuzzDirectBinaryDecoding/seed-valid-record.bin", writeBinaryRecord()); + writeBytes("BinaryDecodingFuzzerInputs/fuzzDirectBinaryDecoding/seed-valid-record-single-node.bin", + writeBinaryRecordWithSingleNode()); writeBytes("DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro", writeContainerFile()); + writeBytes("DataFileReaderFuzzerInputs/fuzzDataFileReaderWithResolution/seed-valid-container.avro", + writeContainerFile()); writeBytes("DataFileReaderFuzzerInputs/fuzzDataFileStream/seed-valid-container.avro", writeContainerFile()); + writeBytes("DataFileReaderFuzzerInputs/fuzzDataFileStreamWithResolution/seed-valid-container.avro", + writeContainerFile()); } private static byte[] writeBinaryRecord() throws IOException { @@ -62,6 +70,15 @@ class SeedCorpusGenerator { return output.toByteArray(); } + private static byte[] writeBinaryRecordWithSingleNode() throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(output, null); + GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(FuzzSupport.BINARY_WRITER_SCHEMA); + writer.write(sampleBinaryRecordWithSingleNode(), encoder); + encoder.flush(); + return output.toByteArray(); + } + private static GenericRecord sampleBinaryRecord() { GenericRecord nodeTail = new GenericData.Record(NODE_SCHEMA); nodeTail.put("value", "tail"); @@ -82,6 +99,22 @@ class SeedCorpusGenerator { return record; } + private static GenericRecord sampleBinaryRecordWithSingleNode() { + GenericRecord node = new GenericData.Record(NODE_SCHEMA); + node.put("value", "solo"); + node.put("next", null); + + GenericRecord record = new GenericData.Record(FuzzSupport.BINARY_WRITER_SCHEMA); + record.put("id", 11L); + record.put("legacyName", "seed-single"); + record.put("createdDate", 1); + record.put("payload", java.nio.ByteBuffer.wrap(new byte[0])); + record.put("hash", + new GenericData.Fixed(FuzzSupport.BINARY_WRITER_SCHEMA.getField("hash").schema(), new byte[] { 9, 8, 7, 6 })); + record.put("node", node); + return record; + } + private static void writeBytes(String relativePath, byte[] data) throws IOException { Path output = RESOURCE_ROOT.resolve(relativePath); Files.createDirectories(output.getParent()); diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/seed-valid-record-single-node.bin b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/seed-valid-record-single-node.bin new file mode 100644 index 0000000000..ab7b989618 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/seed-valid-record-single-node.bin differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzDirectBinaryDecoding/seed-valid-record-single-node.bin b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzDirectBinaryDecoding/seed-valid-record-single-node.bin new file mode 100644 index 0000000000..ab7b989618 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzDirectBinaryDecoding/seed-valid-record-single-node.bin differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro index 1c86ff58f0..4d36604052 100644 Binary files a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReaderWithResolution/seed-valid-container.avro similarity index 86% copy from lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro copy to lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReaderWithResolution/seed-valid-container.avro index 1c86ff58f0..0a1867af3f 100644 Binary files a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReaderWithResolution/seed-valid-container.avro differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStream/seed-valid-container.avro b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStream/seed-valid-container.avro index ee84b3c292..a7b94e7500 100644 Binary files a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStream/seed-valid-container.avro and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStream/seed-valid-container.avro differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStreamWithResolution/seed-valid-container.avro similarity index 86% copy from lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro copy to lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStreamWithResolution/seed-valid-container.avro index 1c86ff58f0..b10d4dfe32 100644 Binary files a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileReader/seed-valid-container.avro and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/DataFileReaderFuzzerInputs/fuzzDataFileStreamWithResolution/seed-valid-container.avro differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/JsonDecodingFuzzerInputs/fuzzJsonDecoding/seed-valid-record-null-extra.json b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/JsonDecodingFuzzerInputs/fuzzJsonDecoding/seed-valid-record-null-extra.json new file mode 100644 index 0000000000..e86e064ae4 --- /dev/null +++ b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/JsonDecodingFuzzerInputs/fuzzJsonDecoding/seed-valid-record-null-extra.json @@ -0,0 +1 @@ +{"id":11,"name":"seed-null","createdDate":1,"active":false,"scores":[0.0],"props":{},"extra":null,"inner":{"x":0,"y":0}} diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/seed-valid-alias-default.avsc b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/seed-valid-alias-default.avsc new file mode 100644 index 0000000000..fd0942ba30 --- /dev/null +++ b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/seed-valid-alias-default.avsc @@ -0,0 +1 @@ +{"type":"record","name":"AliasSeed","fields":[{"name":"legacyName","type":"string"},{"name":"name","aliases":["legacyName"],"type":"string","default":"seed"}]}
