This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 03b940e1c7 fix(arrow-json): validate REE nullability (#10749)
03b940e1c7 is described below
commit 03b940e1c7e3e7af9b636940fb9f1ee195891a1f
Author: WaterWhisperer <[email protected]>
AuthorDate: Fri Aug 21 09:00:12 2026 +0800
fix(arrow-json): validate REE nullability (#10749)
> AI disclosure: Codex (GPT-5.6) assisted with the regression test and
verification. I reviewed and understand all changes and take
responsibility for them.
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes #10478.
# Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
REE arrays have no parent validity buffer, so nulls are valid only when
both the REE field and its values field are nullable.
Follows #10485, which was noted as stalled in [the
issue](https://github.com/apache/arrow-rs/issues/10478#issuecomment-5265435201),
and incorporates its outstanding review.
# What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
- Enforce the combined nullability of the REE field and its values
field.
- Add regression coverage for explicit nulls and missing fields across
all nullability combinations.
# Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
Yes.
- `cargo test -p arrow-json --all-features`
- `cargo clippy --workspace --all-targets --all-features -- -D warnings`
# Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
If there are any breaking changes to public APIs, please call them out.
-->
Yes. The JSON decoder now rejects explicit nulls and missing fields that
violate REE nullability. Such inputs were previously accepted, but there
is no public API change.
---
arrow-json/src/reader/mod.rs | 39 ++++++++++++++++++++++++++++++++++
arrow-json/src/reader/run_end_array.rs | 13 ++++++++----
2 files changed, 48 insertions(+), 4 deletions(-)
diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs
index 0209ede2c7..0c6039d057 100644
--- a/arrow-json/src/reader/mod.rs
+++ b/arrow-json/src/reader/mod.rs
@@ -3669,6 +3669,45 @@ mod tests {
assert_eq!(values.value(2), "y");
}
+ #[test]
+ fn test_read_run_end_encoded_nullability() {
+ for field_nullable in [false, true] {
+ for values_nullable in [false, true] {
+ let ree_type = DataType::RunEndEncoded(
+ Arc::new(Field::new("run_ends", DataType::Int32, false)),
+ Arc::new(Field::new("values", DataType::Utf8,
values_nullable)),
+ );
+ let schema = Arc::new(Schema::new(vec![Field::new("a",
ree_type, field_nullable)]));
+
+ for buf in [
+ r#"{"a": "x"}
+ {"a": null}
+ {"a": "y"}"#,
+ r#"{"a": "x"}
+ {}
+ {"a": "y"}"#,
+ ] {
+ let mut decoder =
ReaderBuilder::new(schema.clone()).build_decoder().unwrap();
+ let result = decoder.decode(buf.as_bytes()).and_then(|_|
decoder.flush());
+
+ if field_nullable && values_nullable {
+ result.expect("REE field and values are both
nullable");
+ } else {
+ let err = result.expect_err(
+ "REE nulls require both the field and values to be
nullable",
+ );
+ assert!(
+ err.to_string().contains(
+ "Encountered nulls in non-nullable values of
RunEndEncoded"
+ ),
+ "unexpected error: {err}"
+ );
+ }
+ }
+ }
+ }
+ }
+
#[test]
fn test_read_run_end_encoded_all_unique() {
let buf = r#"
diff --git a/arrow-json/src/reader/run_end_array.rs
b/arrow-json/src/reader/run_end_array.rs
index 8eb8e82714..df9952f100 100644
--- a/arrow-json/src/reader/run_end_array.rs
+++ b/arrow-json/src/reader/run_end_array.rs
@@ -33,6 +33,7 @@ use crate::reader::{ArrayDecoder, DecoderContext};
pub struct RunEndEncodedArrayDecoder<R> {
data_type: DataType,
decoder: Box<dyn ArrayDecoder>,
+ values_nullable: bool,
phantom: PhantomData<R>,
}
@@ -45,14 +46,13 @@ impl<R: RunEndIndexType> RunEndEncodedArrayDecoder<R> {
let DataType::RunEndEncoded(_, values_field) = data_type else {
unreachable!()
};
- let decoder = ctx.make_decoder(
- values_field.data_type(),
- values_field.is_nullable() || is_nullable,
- )?;
+ let values_nullable = values_field.is_nullable() && is_nullable;
+ let decoder = ctx.make_decoder(values_field.data_type(),
values_nullable)?;
Ok(Self {
data_type: data_type.clone(),
decoder,
+ values_nullable,
phantom: Default::default(),
})
}
@@ -66,6 +66,11 @@ impl<R: RunEndIndexType + Send> ArrayDecoder for
RunEndEncodedArrayDecoder<R> {
}
let flat_array = self.decoder.decode(tape, pos)?;
+ if !self.values_nullable && flat_array.logical_null_count() != 0 {
+ return Err(ArrowError::JsonError(
+ "Encountered nulls in non-nullable values of
RunEndEncoded".to_string(),
+ ));
+ }
let partitions = partition(from_ref(&flat_array))?;
let size = partitions.len();
