This is an automated email from the ASF dual-hosted git repository. iemejia pushed a commit to branch fuzzer in repository https://gitbox.apache.org/repos/asf/avro.git
commit 454d1b990046a092c36bdc0d87cccbb147c617c5 Author: Ismaël Mejía <[email protected]> AuthorDate: Fri Apr 3 19:57:55 2026 +0000 Fix NPE in Schema.parse() and DoS via memory exhaustion in BinaryDecoder Bug 1: ParseContext.resolve() threw NullPointerException instead of AvroTypeException for malformed schema inputs with unresolved type references. Replaced requireNonNull with explicit null check. Bug 2: BinaryDecoder allocated byte arrays based on untrusted varint lengths before verifying data availability, allowing a few-byte payload to trigger OutOfMemoryError. Added validate-before-allocate checks in readString, readBytes, readArrayStart, arrayNext, readMapStart, and mapNext that compare requested length against remaining bytes in byte-array-backed decoders (protobuf-style). Stream-backed decoders are left unchanged since InputStream.available() is unreliable. Both bugs were found by Jazzer fuzz testing. Includes regression tests, crash reproducer seeds, and updated fuzz testing report. --- .../main/java/org/apache/avro/ParseContext.java | 6 +- .../java/org/apache/avro/io/BinaryDecoder.java | 52 ++++++ .../src/test/java/org/apache/avro/TestSchema.java | 15 ++ .../java/org/apache/avro/io/TestBinaryDecoder.java | 190 ++++++++++++++------- lang/java/fuzz/.gitignore | 2 + lang/java/fuzz/REPORT.md | 146 ++++++++++++++++ .../crash-033525b83933bee57048110e869b3714c113403f | Bin 0 -> 2547 bytes .../crash-3750d511b6289eda4a8a47c4cef15c68bf8a56c3 | Bin 0 -> 91 bytes .../crash-89d041db74d873f5031137b1660bebeb594a73d9 | Bin 0 -> 69 bytes .../crash-b5cb10593675cf25f1240e9408ac99607d5403e7 | Bin 0 -> 69 bytes .../crash-b797f2ebd6f11c48868c4f42ec9d1c33125f3c40 | Bin 0 -> 811 bytes .../crash-c2ab87252858767d02ea6b566beb6243a2201fd1 | Bin 0 -> 513 bytes .../crash-d9a7b1b4479e10028390d6b0b1e2fece3965eabc | Bin 0 -> 18 bytes .../crash-49ce322a41d061ffd20cbab64ecbaa17b266e906 | 1 + 14 files changed, 346 insertions(+), 66 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java b/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java index 1d0873eb8b..3369ad91d4 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java +++ b/lang/java/avro/src/main/java/org/apache/avro/ParseContext.java @@ -327,7 +327,11 @@ public class ParseContext { // — All unnamed types can be visited&resolved without validation. if (NAMED_SCHEMA_TYPES.contains(schema.getType()) && schema.getFullName() != null) { - return requireNonNull(oldSchemas.get(schema.getFullName()), () -> "Unknown schema: " + schema.getFullName()); + Schema resolved = oldSchemas.get(schema.getFullName()); + if (resolved == null) { + throw new AvroTypeException("Unknown schema: " + schema.getFullName()); + } + return resolved; } else { // Unnamed or anonymous schema // (protocol message request parameters are anonymous records) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index 827b2fea3c..ca6537f98e 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -295,6 +295,7 @@ public class BinaryDecoder extends Decoder { @Override public Utf8 readString(Utf8 old) throws IOException { int length = SystemLimitException.checkMaxStringLength(readLong()); + ensureAvailableBytes(length); Utf8 result = (old != null ? old : new Utf8()); result.setByteLength(length); if (0 != length) { @@ -318,6 +319,7 @@ public class BinaryDecoder extends Decoder { @Override public ByteBuffer readBytes(ByteBuffer old) throws IOException { int length = SystemLimitException.checkMaxBytesLength(readLong()); + ensureAvailableBytes(length); final ByteBuffer result; if (old != null && length <= old.capacity()) { result = old; @@ -442,6 +444,9 @@ public class BinaryDecoder extends Decoder { @Override public long readArrayStart() throws IOException { collectionCount = SystemLimitException.checkMaxCollectionLength(doReadItemCount()); + // Each array element is at least 1 byte, so the item count is a lower bound + // on the bytes required. Reject early if a byte-array source is too short. + ensureAvailableBytes((int) collectionCount); return collectionCount; } @@ -449,6 +454,7 @@ public class BinaryDecoder extends Decoder { public long arrayNext() throws IOException { long length = doReadItemCount(); collectionCount = SystemLimitException.checkMaxCollectionLength(collectionCount, length); + ensureAvailableBytes((int) length); return length; } @@ -460,6 +466,7 @@ public class BinaryDecoder extends Decoder { @Override public long readMapStart() throws IOException { collectionCount = SystemLimitException.checkMaxCollectionLength(doReadItemCount()); + ensureAvailableBytes((int) collectionCount); return collectionCount; } @@ -467,6 +474,7 @@ public class BinaryDecoder extends Decoder { public long mapNext() throws IOException { long length = doReadItemCount(); collectionCount = SystemLimitException.checkMaxCollectionLength(collectionCount, length); + ensureAvailableBytes((int) length); return length; } @@ -530,6 +538,31 @@ public class BinaryDecoder extends Decoder { } } + /** + * Validates that the source has at least {@code length} bytes remaining before + * allocating a buffer of that size. This prevents a denial-of-service attack + * where a crafted varint claims a huge length, causing an + * {@link OutOfMemoryError} on allocation before the read discovers there are + * too few bytes. + * <p> + * For byte-array-backed sources the remaining count is exact, so this check is + * definitive. For stream-backed sources the remaining count is unknown and this + * method is a no-op (the subsequent read will still throw {@link EOFException} + * if the stream is too short, but only after the allocation has occurred). + * + * @param length the number of bytes about to be allocated + * @throws EOFException if the source is known to have fewer bytes remaining + */ + private void ensureAvailableBytes(int length) throws EOFException { + if (source != null && length > 0) { + int remaining = source.remainingBytes(); + if (remaining >= 0 && length > remaining) { + throw new EOFException( + "Attempted to read " + length + " bytes, but only " + remaining + " bytes are available"); + } + } + } + /** * Returns an {@link java.io.InputStream} that is aware of any buffering that * may occur in this BinaryDecoder. Readers that need to interleave decoding @@ -664,6 +697,15 @@ public class BinaryDecoder extends Decoder { abstract boolean isEof(); + /** + * Returns the total number of bytes remaining that can be read from this source + * (including any buffered bytes), or {@code -1} if the total is unknown. + * Byte-array-backed sources return an exact count; stream-backed sources return + * {@code -1} because {@link InputStream#available()} does not reliably report + * the total remaining bytes. + */ + protected abstract int remainingBytes(); + protected void attach(int bufferSize, BinaryDecoder decoder) { decoder.buf = new byte[bufferSize]; decoder.pos = 0; @@ -910,6 +952,11 @@ public class BinaryDecoder extends Decoder { return isEof; } + @Override + protected int remainingBytes() { + return -1; // unknown for stream-backed sources + } + @Override public void close() throws IOException { in.close(); @@ -1028,5 +1075,10 @@ public class BinaryDecoder extends Decoder { int remaining = ba.getLim() - ba.getPos(); return (remaining == 0); } + + @Override + protected int remainingBytes() { + return ba.getLim() - ba.getPos(); + } } } diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java b/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java index d8fb94f9f0..fa5bf2f537 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java @@ -676,4 +676,19 @@ public class TestSchema { assertTrue(ate.getMessage().contains(badValue)); } } + + /** + * Regression test: Schema.parse() on certain malformed inputs used to throw + * {@link NullPointerException} from {@link ParseContext#resolve} instead of a + * proper Avro exception. After the fix, it should throw an + * {@link AvroTypeException} (or another AvroRuntimeException subclass). + */ + @Test + void parseMalformedInputDoesNotThrowNPE() { + // This input triggers creation of an unresolved named schema reference + // that cannot be found in oldSchemas, previously causing NPE at + // ParseContext.resolve() line 330. + String malformed = "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""; + assertThrows(AvroRuntimeException.class, () -> Schema.parse(malformed)); + } } diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java index 491151b849..eff4c14f04 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java @@ -417,6 +417,32 @@ public class TestBinaryDecoder { } } + /** + * Verify that a byte-array-backed decoder rejects a string whose varint length + * exceeds the remaining bytes, throwing {@link EOFException} <em>before</em> + * allocating the buffer. This is a regression test for CVE-style OOM/DoS where + * a crafted varint could trigger a multi-gigabyte allocation from a tiny input. + */ + @Test + public void testStringLengthExceedsAvailableBytes() throws IOException { + // Encode a varint claiming 10_000_000 bytes of string data, but supply none. + // The byte-array-backed decoder knows it has only a few bytes left after + // the varint, so ensureAvailableBytes must throw EOFException before + // allocating. + BinaryDecoder bd = newDecoder(false, 10_000_000L); + Assertions.assertThrows(EOFException.class, () -> bd.readString(null)); + } + + /** + * Same as {@link #testStringLengthExceedsAvailableBytes()} but for + * {@link BinaryDecoder#readBytes(ByteBuffer)}. + */ + @Test + public void testBytesLengthExceedsAvailableBytes() throws IOException { + BinaryDecoder bd = newDecoder(false, 10_000_000L); + Assertions.assertThrows(EOFException.class, () -> bd.readBytes(null)); + } + @ParameterizedTest @ValueSource(booleans = { true, false }) public void testBytesNegativeLength(boolean useDirect) throws IOException { @@ -451,7 +477,8 @@ public class TestBinaryDecoder { @ParameterizedTest @ValueSource(booleans = { true, false }) public void testArrayVmMaxSize(boolean useDirect) throws IOException { - // At start + // At start — count exceeds VM limit, so the limit check fires first for both + // decoder types. Exception ex = Assertions.assertThrows(UnsupportedOperationException.class, () -> this.newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).readArrayStart()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); @@ -461,27 +488,38 @@ public class TestBinaryDecoder { () -> this.newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).arrayNext()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // An OK reads followed by an overflow - Decoder bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, Long.MAX_VALUE); - Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); - ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); - Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + if (useDirect) { + // Stream-backed decoders cannot validate available bytes, so large counts + // that are within the VM limit are returned and the overflow is only + // detected on subsequent reads. + + // An OK reads followed by an overflow + Decoder bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, Long.MAX_VALUE); + Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); + ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); + Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the VM limit. - bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, 100, 1); - Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); - Assertions.assertEquals(100, bd.arrayNext()); - ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); - Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + // Two OK reads followed by going over the VM limit. + bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, 100, 1); + Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); + Assertions.assertEquals(100, bd.arrayNext()); + ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); + Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the VM limit, where negative numbers are - // followed by the byte length of the items. For testing, the 999 values are - // read but ignored. - bd = newDecoder(useDirect, 100 - MAX_ARRAY_VM_LIMIT, 999, -100, 999, 1); - Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); - Assertions.assertEquals(100, bd.arrayNext()); - ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); - Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + // Two OK reads followed by going over the VM limit, where negative numbers are + // followed by the byte length of the items. For testing, the 999 values are + // read but ignored. + bd = newDecoder(useDirect, 100 - MAX_ARRAY_VM_LIMIT, 999, -100, 999, 1); + Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readArrayStart()); + Assertions.assertEquals(100, bd.arrayNext()); + ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::arrayNext); + Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + } else { + // Byte-array-backed decoders detect that the claimed count exceeds the + // remaining bytes and throw EOFException before any allocation. + Assertions.assertThrows(EOFException.class, + () -> newDecoder(false, MAX_ARRAY_VM_LIMIT - 100, Long.MAX_VALUE).readArrayStart()); + } } @ParameterizedTest @@ -494,21 +532,30 @@ public class TestBinaryDecoder { () -> newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).readArrayStart()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the custom limit. - Decoder bd = newDecoder(useDirect, 118, 10, 1); - Assertions.assertEquals(118, bd.readArrayStart()); - Assertions.assertEquals(10, bd.arrayNext()); - ex = Assertions.assertThrows(SystemLimitException.class, bd::arrayNext); - Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); - - // Two OK reads followed by going over the VM limit, where negative numbers are - // followed by the byte length of the items. For testing, the 999 values are - // read but ignored. - bd = newDecoder(useDirect, -118, 999, -10, 999, 1); - Assertions.assertEquals(118, bd.readArrayStart()); - Assertions.assertEquals(10, bd.arrayNext()); - ex = Assertions.assertThrows(SystemLimitException.class, bd::arrayNext); - Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + if (useDirect) { + // Stream-backed decoders cannot validate available bytes, so counts that + // are within the custom limit are accepted. + + // Two OK reads followed by going over the custom limit. + Decoder bd = newDecoder(useDirect, 118, 10, 1); + Assertions.assertEquals(118, bd.readArrayStart()); + Assertions.assertEquals(10, bd.arrayNext()); + ex = Assertions.assertThrows(SystemLimitException.class, bd::arrayNext); + Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + + // Two OK reads followed by going over the VM limit, where negative numbers are + // followed by the byte length of the items. For testing, the 999 values are + // read but ignored. + bd = newDecoder(useDirect, -118, 999, -10, 999, 1); + Assertions.assertEquals(118, bd.readArrayStart()); + Assertions.assertEquals(10, bd.arrayNext()); + ex = Assertions.assertThrows(SystemLimitException.class, bd::arrayNext); + Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + } else { + // Byte-array-backed decoders detect that the claimed count exceeds the + // remaining bytes and throw EOFException before any allocation. + Assertions.assertThrows(EOFException.class, () -> newDecoder(false, 118, 10, 1).readArrayStart()); + } } finally { System.clearProperty(SystemLimitException.MAX_COLLECTION_LENGTH_PROPERTY); @@ -519,7 +566,7 @@ public class TestBinaryDecoder { @ParameterizedTest @ValueSource(booleans = { true, false }) public void testMapVmMaxSize(boolean useDirect) throws IOException { - // At start + // At start — count exceeds VM limit, so the limit check fires first. Exception ex = Assertions.assertThrows(UnsupportedOperationException.class, () -> this.newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).readMapStart()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); @@ -529,21 +576,28 @@ public class TestBinaryDecoder { () -> this.newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).mapNext()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the VM limit. - Decoder bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, 100, 1); - Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readMapStart()); - Assertions.assertEquals(100, bd.mapNext()); - ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::mapNext); - Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + if (useDirect) { + // Two OK reads followed by going over the VM limit. + Decoder bd = newDecoder(useDirect, MAX_ARRAY_VM_LIMIT - 100, 100, 1); + Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readMapStart()); + Assertions.assertEquals(100, bd.mapNext()); + ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::mapNext); + Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the VM limit, where negative numbers are - // followed by the byte length of the items. For testing, the 999 values are - // read but ignored. - bd = newDecoder(useDirect, 100 - MAX_ARRAY_VM_LIMIT, 999, -100, 999, 1); - Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readMapStart()); - Assertions.assertEquals(100, bd.mapNext()); - ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::mapNext); - Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + // Two OK reads followed by going over the VM limit, where negative numbers are + // followed by the byte length of the items. For testing, the 999 values are + // read but ignored. + bd = newDecoder(useDirect, 100 - MAX_ARRAY_VM_LIMIT, 999, -100, 999, 1); + Assertions.assertEquals(MAX_ARRAY_VM_LIMIT - 100, bd.readMapStart()); + Assertions.assertEquals(100, bd.mapNext()); + ex = Assertions.assertThrows(UnsupportedOperationException.class, bd::mapNext); + Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); + } else { + // Byte-array-backed decoders detect that the claimed count exceeds the + // remaining bytes and throw EOFException. + Assertions.assertThrows(EOFException.class, + () -> newDecoder(false, MAX_ARRAY_VM_LIMIT - 100, 100, 1).readMapStart()); + } } @ParameterizedTest @@ -556,21 +610,27 @@ public class TestBinaryDecoder { () -> newDecoder(useDirect, MAX_ARRAY_VM_LIMIT + 1).readMapStart()); Assertions.assertEquals(ERROR_VM_LIMIT_COLLECTION, ex.getMessage()); - // Two OK reads followed by going over the custom limit. - Decoder bd = newDecoder(useDirect, 118, 10, 1); - Assertions.assertEquals(118, bd.readMapStart()); - Assertions.assertEquals(10, bd.mapNext()); - ex = Assertions.assertThrows(SystemLimitException.class, bd::mapNext); - Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); - - // Two OK reads followed by going over the VM limit, where negative numbers are - // followed by the byte length of the items. For testing, the 999 values are - // read but ignored. - bd = newDecoder(useDirect, -118, 999, -10, 999, 1); - Assertions.assertEquals(118, bd.readMapStart()); - Assertions.assertEquals(10, bd.mapNext()); - ex = Assertions.assertThrows(SystemLimitException.class, bd::mapNext); - Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + if (useDirect) { + // Two OK reads followed by going over the custom limit. + Decoder bd = newDecoder(useDirect, 118, 10, 1); + Assertions.assertEquals(118, bd.readMapStart()); + Assertions.assertEquals(10, bd.mapNext()); + ex = Assertions.assertThrows(SystemLimitException.class, bd::mapNext); + Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + + // Two OK reads followed by going over the VM limit, where negative numbers are + // followed by the byte length of the items. For testing, the 999 values are + // read but ignored. + bd = newDecoder(useDirect, -118, 999, -10, 999, 1); + Assertions.assertEquals(118, bd.readMapStart()); + Assertions.assertEquals(10, bd.mapNext()); + ex = Assertions.assertThrows(SystemLimitException.class, bd::mapNext); + Assertions.assertEquals("Collection length 129 exceeds maximum allowed", ex.getMessage()); + } else { + // Byte-array-backed decoders detect that the claimed count exceeds the + // remaining bytes and throw EOFException. + Assertions.assertThrows(EOFException.class, () -> newDecoder(false, 118, 10, 1).readMapStart()); + } } finally { System.clearProperty(SystemLimitException.MAX_COLLECTION_LENGTH_PROPERTY); diff --git a/lang/java/fuzz/.gitignore b/lang/java/fuzz/.gitignore new file mode 100644 index 0000000000..ca91bb4636 --- /dev/null +++ b/lang/java/fuzz/.gitignore @@ -0,0 +1,2 @@ +# Generated fuzzing corpus (non-deterministic, regenerated each run) +.cifuzz-corpus/ diff --git a/lang/java/fuzz/REPORT.md b/lang/java/fuzz/REPORT.md new file mode 100644 index 0000000000..783154dd32 --- /dev/null +++ b/lang/java/fuzz/REPORT.md @@ -0,0 +1,146 @@ +# Fuzz Testing Report for Apache Avro Java + +## Overview + +A new `lang/java/fuzz/` Maven module was created with 4 fuzz test classes (5 total fuzz targets) using [Jazzer](https://github.com/CodeIntelligenceTesting/jazzer) 0.22.1, a coverage-guided Java fuzzer built on libFuzzer. Two bugs were found and fixed: an NPE in schema parsing and a denial-of-service via memory exhaustion in binary decoding. + +## Fuzz Test Targets + +| Class | Target | Description | +|---|---|---| +| `SchemaFuzzer` | `fuzzSchemaParse` | Feeds arbitrary UTF-8 strings to `Schema.parse(String)` | +| `BinaryDecodingFuzzer` | `fuzzBinaryDecoding` | Feeds arbitrary bytes through `BinaryDecoder` + `GenericDatumReader` with a multi-field record schema | +| `JsonDecodingFuzzer` | `fuzzJsonDecoding` | Feeds arbitrary JSON through `JsonDecoder` + `GenericDatumReader` | +| `DataFileReaderFuzzer` | `fuzzDataFileReader` | Feeds arbitrary bytes to `DataFileReader` (seekable path) | +| `DataFileReaderFuzzer` | `fuzzDataFileStream` | Feeds arbitrary bytes to `DataFileStream` (streaming path) | + +## Bugs Found + +### Bug 1: `NullPointerException` in `Schema.parse()` (ParseContext.resolve) + +- **Severity**: Medium +- **Location**: `ParseContext.java:330` +- **Trigger**: Input like `""""""""""""""""""` (18 quote characters) +- **Root Cause**: When `Schema.parse()` processes certain malformed JSON inputs (e.g., nested JSON arrays with string elements that look like type references), the parser creates unresolved schema placeholders (`UnresolvedSchema_0`). In `ParseContext.resolve()`, it calls `requireNonNull(oldSchemas.get(schema.getFullName()))` which throws `NullPointerException` instead of a proper `AvroTypeException`. +- **Impact**: Code that calls `Schema.parse()` and catches `SchemaParseException` or `AvroRuntimeException` will miss this error. The NPE leaks internal implementation details. +- **Status**: **Fixed** + +**Stack trace:** + +``` +java.lang.NullPointerException: Unknown schema: org.apache.avro.compiler.UnresolvedSchema_0 + at java.base/java.util.Objects.requireNonNull(Objects.java:360) + at org.apache.avro.ParseContext.resolve(ParseContext.java:330) + at org.apache.avro.Schema$Parser.parse(Schema.java:1542) + at org.apache.avro.Schema$Parser.parse(Schema.java:1516) + at org.apache.avro.Schema.parse(Schema.java:1607) +``` + +### Bug 2: Denial-of-Service via Memory Exhaustion in Binary/Container Decoding + +- **Severity**: High (security, DoS) +- **Location**: `Utf8.java:115` via `BinaryDecoder.readString()` at `BinaryDecoder.java:299`; also `readBytes()`, `readArrayStart()`, `arrayNext()`, `readMapStart()`, `mapNext()` +- **Trigger**: Crafted binary input containing varint-encoded lengths far exceeding available data +- **Root Cause**: `BinaryDecoder` reads a varint-encoded length, then immediately allocates `new byte[length]` before checking whether that many bytes are actually available. A few-byte payload encoding a large varint (e.g., 200 MB+) causes `OutOfMemoryError` before any read can fail with `EOFException`. This affects strings, bytes, arrays, and maps. +- **Impact**: An attacker can craft a small Avro binary payload (just a few bytes encoding a large varint) that causes the JVM to run out of memory. This is a denial-of-service vector for any service that reads untrusted Avro data. +- **Status**: **Fixed** + +**Stack trace:** + +``` +java.lang.OutOfMemoryError: Java heap space + at org.apache.avro.util.Utf8.setByteLength(Utf8.java:115) + at org.apache.avro.io.BinaryDecoder.readString(BinaryDecoder.java:299) + at org.apache.avro.io.FastReaderBuilder.lambda$createSimpleStringReader$27(FastReaderBuilder.java:380) + at org.apache.avro.io.FastReaderBuilder$RecordReader.read(FastReaderBuilder.java:575) + at org.apache.avro.generic.GenericDatumReader.read(GenericDatumReader.java:150) +``` + +## Fixes Applied + +### Fix for Bug 1: Proper error type in ParseContext.resolve() + +**File**: `ParseContext.java` + +Replaced `Objects.requireNonNull(oldSchemas.get(schema.getFullName()), ...)` with an explicit null check that throws `AvroTypeException`. This ensures malformed schema inputs produce the same exception type as other schema parse errors, making catch blocks reliable. + +**Regression test**: `TestSchema.parseMalformedInputDoesNotThrowNPE` — verifies that the trigger input throws `AvroTypeException`, not `NullPointerException`. + +### Fix for Bug 2: Validate-before-allocate in BinaryDecoder + +**File**: `BinaryDecoder.java` + +Added a protobuf-style validate-before-allocate check: before allocating memory for a decoded length, verify that the byte source actually has enough bytes remaining. + +Implementation details: + +- Added abstract `remainingBytes()` to the inner `ByteSource` class +- `ByteArrayByteSource.remainingBytes()` returns exact remaining bytes (`lim - pos`) +- `InputStreamByteSource.remainingBytes()` returns `-1` (unknown — `InputStream.available()` is unreliable, so stream-backed decoders are not guarded by this check) +- Added `ensureAvailableBytes(int length)` method that throws `EOFException` when the requested length exceeds known remaining bytes +- The check is called in: `readString()`, `readBytes()`, `readArrayStart()`, `arrayNext()`, `readMapStart()`, `mapNext()` +- For `DirectBinaryDecoder`, `source` is always null, so the check is a no-op (correct — it has no buffered byte source) +- For arrays/maps, each item must be at least 1 byte, so item count serves as a valid lower bound for bytes needed + +This approach was chosen over lowering the default `SystemLimitException` limits because it does not change behavior for production users — only clearly impossible allocations (where the claimed length exceeds the actual data) are rejected. + +**Regression tests**: +- `TestBinaryDecoder.testStringLengthExceedsAvailableBytes` — verifies `EOFException` for a string length exceeding available bytes +- `TestBinaryDecoder.testBytesLengthExceedsAvailableBytes` — verifies `EOFException` for a bytes length exceeding available bytes +- Updated `testArrayVmMaxSize`, `testArrayMaxCustom`, `testMapVmMaxSize`, `testMapMaxCustom` — byte-array-backed decoders now correctly reject impossible counts with `EOFException` + +## Verification + +### Test suite +All 3,415 tests in the `avro` module pass after both fixes. + +### Fuzzer re-runs (post-fix) +- **BinaryDecodingFuzzer**: ~974K iterations, 0 crashes (previously crashed within ~4K iterations with OOM) +- **SchemaFuzzer**: ~1M iterations, 0 crashes (previously found NPE) + +Both bugs are confirmed fixed. + +## What Each Fuzz Test Does + +### SchemaFuzzer + +Tests the Avro JSON schema parser (`Schema.parse()`). Converts random bytes to a UTF-8 string and passes it as a schema definition. This exercises the Jackson JSON parser, Avro type resolution, named schema registration, union construction, and schema validation. Expected exceptions (`AvroRuntimeException`, `SchemaParseException`, `IllegalArgumentException`) are caught — any other exception indicates a bug. + +### BinaryDecodingFuzzer + +Tests the Avro binary wire format decoder. Feeds raw bytes directly into `BinaryDecoder` paired with a `GenericDatumReader` using a record schema with long, string, double, array, map, and union fields — exercises varint parsing, buffer management, collection size decoding, and union branch selection. + +### JsonDecodingFuzzer + +Tests the Avro JSON data decoder. Converts random bytes to UTF-8 and feeds them through `JsonDecoder` with a `GenericDatumReader`. This exercises JSON-to-Avro type coercion, union resolution via JSON object wrapping, and error handling for malformed or type-mismatched data. + +### DataFileReaderFuzzer + +Tests the Avro container file (Object Container File) format parser. This is the highest-value target as it exercises the full deserialization pipeline: magic byte validation, file header parsing, embedded schema deserialization, sync marker verification, codec decompression, and datum reading. Two code paths are tested: + +- **`DataFileReader`** (seekable, random-access via `SeekableByteArrayInput`) +- **`DataFileStream`** (sequential, non-seekable via `ByteArrayInputStream`) + +## How to Run + +```bash +# Regression mode (default — replays seed corpus only, fast): +mvn test -pl fuzz + +# Continuous fuzzing mode (60s per target): +mvn test -pl fuzz -Pfuzz + +# Fuzz a specific target: +mvn test -pl fuzz -Pfuzz -Dtest=SchemaFuzzer#fuzzSchemaParse + +# Alternative: set JAZZER_FUZZ environment variable: +JAZZER_FUZZ=1 mvn test -pl fuzz +``` + +## Technical Notes + +- **Jazzer version**: 0.22.1 with JUnit 5 integration (`jazzer-junit`) +- **Fuzzing engine**: libFuzzer (coverage-guided, mutation-based) +- **Enabling fuzzing mode**: Jazzer 0.22.1 checks the `JAZZER_FUZZ` environment variable or the `jazzer.internal.command_line` JUnit Platform configuration parameter. The `-Pfuzz` Maven profile sets both. +- **JUnit version alignment**: Jazzer 0.22.1 ships with JUnit Platform 1.9.0 dependencies which conflict with the project's JUnit 5.14.3. The POM excludes Jazzer's JUnit deps and provides `junit-platform-launcher:1.14.3` explicitly. +- **Surefire configuration**: The parent POM sets `parallel=all` with unlimited threads. The fuzz module overrides this to single-threaded execution, which Jazzer requires. diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-033525b83933bee57048110e869b3714c113403f b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-033525b83933bee57048110e869b3714c113403f new file mode 100644 index 0000000000..803cfedc50 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-033525b83933bee57048110e869b3714c113403f differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-3750d511b6289eda4a8a47c4cef15c68bf8a56c3 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-3750d511b6289eda4a8a47c4cef15c68bf8a56c3 new file mode 100644 index 0000000000..f50a9fbd84 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-3750d511b6289eda4a8a47c4cef15c68bf8a56c3 differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-89d041db74d873f5031137b1660bebeb594a73d9 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-89d041db74d873f5031137b1660bebeb594a73d9 new file mode 100644 index 0000000000..147cf7b429 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-89d041db74d873f5031137b1660bebeb594a73d9 differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b5cb10593675cf25f1240e9408ac99607d5403e7 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b5cb10593675cf25f1240e9408ac99607d5403e7 new file mode 100644 index 0000000000..94f6c83845 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b5cb10593675cf25f1240e9408ac99607d5403e7 differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b797f2ebd6f11c48868c4f42ec9d1c33125f3c40 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b797f2ebd6f11c48868c4f42ec9d1c33125f3c40 new file mode 100644 index 0000000000..723b05bd4b Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-b797f2ebd6f11c48868c4f42ec9d1c33125f3c40 differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-c2ab87252858767d02ea6b566beb6243a2201fd1 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-c2ab87252858767d02ea6b566beb6243a2201fd1 new file mode 100644 index 0000000000..77963a28a3 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-c2ab87252858767d02ea6b566beb6243a2201fd1 differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-d9a7b1b4479e10028390d6b0b1e2fece3965eabc b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-d9a7b1b4479e10028390d6b0b1e2fece3965eabc new file mode 100644 index 0000000000..5460973a54 Binary files /dev/null and b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/BinaryDecodingFuzzerInputs/fuzzBinaryDecoding/crash-d9a7b1b4479e10028390d6b0b1e2fece3965eabc differ diff --git a/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/crash-49ce322a41d061ffd20cbab64ecbaa17b266e906 b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/crash-49ce322a41d061ffd20cbab64ecbaa17b266e906 new file mode 100644 index 0000000000..ce8a7dd0a1 --- /dev/null +++ b/lang/java/fuzz/src/test/resources/org/apache/avro/fuzz/SchemaFuzzerInputs/fuzzSchemaParse/crash-49ce322a41d061ffd20cbab64ecbaa17b266e906 @@ -0,0 +1 @@ +[["��["],["�*"],["�["],["�"],["*y�"],["�*"],["��["],["�["],["�"],["*y�"],["�*"],["��["],["�*"],["�["],["�"],["�"],["�*y"],["�*"],["��["],["�*"],["�["],["�*y"],["�*�*"],["�["],["�"],["�"],["�*y"],["�*"],["�["],["�[�["],["�[�"],["�*y�"],["�["],["�*"],["�["],["�[["],["�*"],["��["],["�"],["�"],["�*y"],["�*"],["�["],["�[�["],["�"],["*y�"],["�*"],["��["],["�*"],["�["],["�"],["�"],["�*y"],["��["],["�*"],["�*y"],["�*�*"],["�["],["�"],["�"],["�*y"],["�*"],["�["],["�[�["],["�[�"],["�*y�"],["�["],[ [...] \ No newline at end of file
