This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new b84fc5cfa9 Json decoder factory (#10670)
b84fc5cfa9 is described below
commit b84fc5cfa9bbc8698d2487a444c6baaa489351f7
Author: Haresh Khanna <[email protected]>
AuthorDate: Wed Sep 2 11:56:46 2026 +0100
Json decoder factory (#10670)
# Which issue does this PR close?
- Related: #9021, #9272 (prior art, both went stale); a step toward
#8987 but does not close it.
- Closes https://github.com/apache/arrow-rs/issues/10739
# Rationale for this change
- The JSON writer has been extensible since #7015 (`EncoderFactory`,
plus a public `make_encoder` to delegate to defaults). The reader has no
equivalent.
- Small now because #9272's prefactors already merged (#9266, #9270,
#9271): `DecoderContext` already sits on `main`.
# What changes are included in this PR?
- `TapeElement` `#[non_exhaustive]`, documenting numbers as `Number`
(JSON text) or native `i32`/`i64`/`f32`/`f64` (serde path), 64-bit
spanning two elements.
- `ArrayDecoder` public; `pos` holds one tape index per output row.
- New `DecoderFactory`, consulted before the reader's dispatch;
`Ok(None)` accepts the default.
- `DecoderContext::make_decoder` public, so a factory can delegate to
the decoder the reader would otherwise use - without it, overriding a
nested type means reimplementing its children (@scovich's point on
#9021).
- `ReaderBuilder::with_decoder_factory`.
# Are these changes tested?
Yes. Unit tests and a doctest decoding `Binary` from a JSON int array -
the inverse of the existing `EncoderFactory` doctest, so the two
round-trip, no new dependency.
# Are there any user-facing changes?
New: `arrow_json::{Tape, TapeElement, ArrayDecoder, DecoderFactory}`,
`DecoderContext::{make_decoder, decoder_factory}`,
`ReaderBuilder::with_decoder_factory`.
---
arrow-json/src/lib.rs | 4 +-
arrow-json/src/reader/list_array.rs | 4 +-
arrow-json/src/reader/map_array.rs | 10 +-
arrow-json/src/reader/mod.rs | 413 +++++++++++++++++++++++++++++++--
arrow-json/src/reader/run_end_array.rs | 2 +-
arrow-json/src/reader/struct_array.rs | 2 +-
arrow-json/src/reader/tape.rs | 11 +
7 files changed, 417 insertions(+), 29 deletions(-)
diff --git a/arrow-json/src/lib.rs b/arrow-json/src/lib.rs
index ee5d7df73a..e5c77bcd73 100644
--- a/arrow-json/src/lib.rs
+++ b/arrow-json/src/lib.rs
@@ -91,7 +91,9 @@
pub mod reader;
pub mod writer;
-pub use self::reader::{Reader, ReaderBuilder};
+pub use self::reader::{
+ ArrayDecoder, DecoderContext, DecoderFactory, Reader, ReaderBuilder, Tape,
TapeElement,
+};
pub use self::writer::{
ArrayWriter, Encoder, EncoderFactory, EncoderOptions, LineDelimitedWriter,
Writer,
WriterBuilder,
diff --git a/arrow-json/src/reader/list_array.rs
b/arrow-json/src/reader/list_array.rs
index fc6af59079..6b62ae3948 100644
--- a/arrow-json/src/reader/list_array.rs
+++ b/arrow-json/src/reader/list_array.rs
@@ -51,7 +51,7 @@ impl<O: OffsetSizeTrait, const IS_VIEW: bool>
ListLikeArrayDecoder<O, IS_VIEW> {
(true, DataType::LargeListView(f)) if O::IS_LARGE => f,
_ => unreachable!(),
};
- let decoder = ctx.make_decoder(field.data_type(),
field.is_nullable())?;
+ let decoder = ctx.make_decoder(field, field.is_nullable())?;
Ok(Self {
field: field.clone(),
@@ -148,7 +148,7 @@ impl FixedSizeListArrayDecoder {
DataType::FixedSizeList(f, s) => (f, *s),
_ => unreachable!(),
};
- let decoder = ctx.make_decoder(field.data_type(),
field.is_nullable())?;
+ let decoder = ctx.make_decoder(field, field.is_nullable())?;
Ok(Self {
field: field.clone(),
diff --git a/arrow-json/src/reader/map_array.rs
b/arrow-json/src/reader/map_array.rs
index cd8f868e31..14103b9448 100644
--- a/arrow-json/src/reader/map_array.rs
+++ b/arrow-json/src/reader/map_array.rs
@@ -59,14 +59,8 @@ impl MapArrayDecoder {
}
};
- let keys = ctx.make_decoder(
- key_value_fields[0].data_type(),
- key_value_fields[0].is_nullable(),
- )?;
- let values = ctx.make_decoder(
- key_value_fields[1].data_type(),
- key_value_fields[1].is_nullable(),
- )?;
+ let keys = ctx.make_decoder(&key_value_fields[0],
key_value_fields[0].is_nullable())?;
+ let values = ctx.make_decoder(&key_value_fields[1],
key_value_fields[1].is_nullable())?;
Ok(Self {
entries_field,
diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs
index f562295e2d..d0ca52301a 100644
--- a/arrow-json/src/reader/mod.rs
+++ b/arrow-json/src/reader/mod.rs
@@ -133,7 +133,6 @@
//! ```
//!
-use std::borrow::Cow;
use std::io::BufRead;
use std::sync::Arc;
@@ -141,7 +140,7 @@ use arrow_array::cast::AsArray;
use arrow_array::timezone::Tz;
use arrow_array::types::*;
use arrow_array::{ArrayRef, RecordBatch, RecordBatchReader, downcast_integer};
-use arrow_schema::{ArrowError, DataType, FieldRef, Schema, SchemaRef,
TimeUnit};
+use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema, SchemaRef,
TimeUnit};
use chrono::Utc;
use serde_core::Serialize;
@@ -161,10 +160,12 @@ use
crate::reader::run_end_array::RunEndEncodedArrayDecoder;
use crate::reader::string_array::StringArrayDecoder;
use crate::reader::string_view_array::StringViewArrayDecoder;
use crate::reader::struct_array::StructArrayDecoder;
-use crate::reader::tape::{Tape, TapeDecoder, TapeDecoderOptions};
+use crate::reader::tape::{TapeDecoder, TapeDecoderOptions};
use crate::reader::timestamp_array::TimestampArrayDecoder;
pub use schema::*;
+// `mod tape` stays private so `TapeDecoder` does not become public API
+pub use tape::{Tape, TapeElement};
pub use value_iter::ValueIter;
mod binary_array;
@@ -194,7 +195,7 @@ pub struct ReaderBuilder {
is_field: bool,
struct_mode: StructMode,
flatten_top_level_arrays: bool,
-
+ decoder_factory: Option<Arc<dyn DecoderFactory>>,
schema: SchemaRef,
}
@@ -216,6 +217,7 @@ impl ReaderBuilder {
is_field: false,
struct_mode: Default::default(),
flatten_top_level_arrays: false,
+ decoder_factory: None,
schema,
}
}
@@ -259,6 +261,7 @@ impl ReaderBuilder {
is_field: true,
struct_mode: Default::default(),
flatten_top_level_arrays: false,
+ decoder_factory: None,
schema: Arc::new(Schema::new([field.into()])),
}
}
@@ -341,6 +344,14 @@ impl ReaderBuilder {
}
}
+ /// Set a hook for customizing decoding behavior. See [`DecoderFactory`].
+ pub fn with_decoder_factory(self, decoder_factory: Arc<dyn
DecoderFactory>) -> Self {
+ Self {
+ decoder_factory: Some(decoder_factory),
+ ..self
+ }
+ }
+
/// Create a [`Reader`] with the provided [`BufRead`]
pub fn build<R: BufRead>(self, reader: R) -> Result<Reader<R>, ArrowError>
{
Ok(Reader {
@@ -351,13 +362,14 @@ impl ReaderBuilder {
/// Create a [`Decoder`]
pub fn build_decoder(self) -> Result<Decoder, ArrowError> {
- let (data_type, nullable) = if self.is_field {
- let field = &self.schema.fields[0];
- let data_type = Cow::Borrowed(field.data_type());
- (data_type, field.is_nullable())
+ let (field, nullable) = if self.is_field {
+ let field = self.schema.fields[0].clone();
+ let nullable = field.is_nullable();
+ (field, nullable)
} else {
- let data_type =
Cow::Owned(DataType::Struct(self.schema.fields.clone()));
- (data_type, false)
+ // The root has no field of its own; synthesize a nameless one
+ let data_type = DataType::Struct(self.schema.fields.clone());
+ (Arc::new(Field::new("", data_type, false)), false)
};
let ctx = DecoderContext {
@@ -365,8 +377,9 @@ impl ReaderBuilder {
strict_mode: self.strict_mode,
struct_mode: self.struct_mode,
ignore_type_conflicts: self.ignore_type_conflicts,
+ decoder_factory: self.decoder_factory,
};
- let decoder = ctx.make_decoder(data_type.as_ref(), nullable)?;
+ let decoder = ctx.make_decoder(&field, nullable)?;
let num_fields = self.schema.flattened_fields().len();
@@ -735,11 +748,208 @@ impl Decoder {
}
}
-trait ArrayDecoder: Send {
+/// Decodes a column of JSON values from a [`Tape`] into an [`ArrayRef`]
+///
+/// Implement alongside [`DecoderFactory`] to override or add support for a
type.
+pub trait ArrayDecoder: Send {
/// Decode elements from `tape` starting at the indexes contained in `pos`
+ ///
+ /// `pos` contains one tape index per output row, so the returned array
must have
+ /// exactly `pos.len()` elements and the field's data type. A row's value
may be
+ /// [`TapeElement::Null`].
fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError>;
}
+/// A trait to create custom decoders for specific data types.
+///
+/// Overrides the reader's decoder for a data type, or adds support for one it
does not
+/// handle. The reader-side counterpart of [`EncoderFactory`]; register an
+/// implementation with [`ReaderBuilder::with_decoder_factory`].
+///
+/// # Examples
+///
+/// Decodes `Binary` from a JSON array of integers rather than the default hex
string.
+///
+/// ```
+/// use std::sync::Arc;
+/// use arrow_array::{Array, ArrayRef, BinaryArray};
+/// use arrow_array::types::Float64Type;
+/// use arrow_array::cast::AsArray;
+/// use arrow_json::reader::{ArrayDecoder, DecoderContext, DecoderFactory,
Tape, TapeElement};
+/// use arrow_json::ReaderBuilder;
+/// use arrow_schema::{ArrowError, DataType, Field, FieldRef, Fields, Schema};
+/// use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
+/// use arrow_array::StringArray;
+///
+/// /// Decodes `[104, 105]` into the bytes `b"hi"`
+/// struct IntArrayBinaryDecoder;
+///
+/// impl ArrayDecoder for IntArrayBinaryDecoder {
+/// fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError> {
+/// let mut values: Vec<Option<Vec<u8>>> =
Vec::with_capacity(pos.len());
+/// for p in pos {
+/// match tape.get(*p) {
+/// TapeElement::Null => values.push(None),
+/// TapeElement::StartList(end) => {
+/// let mut bytes = Vec::new();
+/// let mut cur = p + 1;
+/// while cur < end {
+/// match tape.get(cur) {
+/// // JSON text yields `Number`; serde yields
`I32`
+/// TapeElement::Number(idx) => {
+/// let s = tape.get_string(idx);
+/// bytes.push(s.parse::<u8>().map_err(|e| {
+/// ArrowError::JsonError(format!("invalid
byte {s}: {e}"))
+/// })?);
+/// }
+/// TapeElement::I32(v) => bytes.push(v as u8),
+/// _ => return Err(tape.error(cur, "byte")),
+/// }
+/// cur = tape.next(cur, "byte")?;
+/// }
+/// values.push(Some(bytes));
+/// }
+/// _ => return Err(tape.error(*p, "list of bytes")),
+/// }
+/// }
+/// Ok(Arc::new(BinaryArray::from_iter(values.iter().map(|v|
v.as_deref()))))
+/// }
+/// }
+///
+/// /// Upper-cases whatever the reader's own decoder produced
+/// struct ShoutDecoder(Box<dyn ArrayDecoder>);
+///
+/// impl ArrayDecoder for ShoutDecoder {
+/// fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError> {
+/// let inner = self.0.decode(tape, pos)?;
+/// let values = inner.as_string::<i32>();
+/// Ok(Arc::new(StringArray::from_iter(
+/// values.iter().map(|v| v.map(str::to_uppercase)),
+/// )))
+/// }
+/// }
+///
+/// #[derive(Debug)]
+/// struct IntArrayBinaryDecoderFactory;
+///
+/// impl DecoderFactory for IntArrayBinaryDecoderFactory {
+/// fn make_default_decoder(
+/// &self,
+/// ctx: &DecoderContext,
+/// field: &FieldRef,
+/// is_nullable: bool,
+/// ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
+/// // Selection can key off metadata, e.g. to recognise an extension
type, and
+/// // build on the reader's own decoder for the very same field
+/// if
field.metadata().get(EXTENSION_TYPE_NAME_KEY).map(String::as_str)
+/// == Some("apache.shout")
+/// {
+/// let inner = ctx.make_builtin_decoder(field, is_nullable)?;
+/// return Ok(Some(Box::new(ShoutDecoder(inner))));
+/// }
+///
+/// match field.data_type() {
+/// DataType::Binary => Ok(Some(Box::new(IntArrayBinaryDecoder))),
+/// // Returning `None` uses the reader's default decoder
+/// _ => Ok(None),
+/// }
+/// }
+/// }
+///
+/// let nested = Fields::from(vec![Field::new("inner", DataType::Binary,
true)]);
+/// let schema = Arc::new(Schema::new(vec![
+/// Field::new("bytes", DataType::Binary, true),
+/// Field::new("float", DataType::Float64, true),
+/// Field::new("nested", DataType::Struct(nested), true),
+/// Field::new("shout", DataType::Utf8, true)
+/// .with_metadata([(EXTENSION_TYPE_NAME_KEY, "apache.shout")]),
+/// ]));
+///
+/// let json = r#"{"bytes": [104, 105], "float": 1.0, "nested": {"inner":
[104, 105]}, "shout": "hi"}
+/// {"float": 2.3}
+/// {"bytes": [98], "nested": {"inner": [98]}}
+/// "#;
+///
+/// let batch = ReaderBuilder::new(schema)
+/// .with_decoder_factory(Arc::new(IntArrayBinaryDecoderFactory))
+/// .build(json.as_bytes())
+/// .unwrap()
+/// .next()
+/// .unwrap()
+/// .unwrap();
+///
+/// let bytes = batch.column(0).as_binary::<i32>();
+/// assert_eq!(bytes.value(0), b"hi");
+/// assert!(bytes.is_null(1));
+/// assert_eq!(bytes.value(2), b"b");
+///
+/// // The override applies wherever `Binary` appears, including nested, while
types
+/// // the factory declines are decoded as usual
+/// let inner = batch.column(2).as_struct().column(0).as_binary::<i32>();
+/// assert_eq!(inner.value(0), b"hi");
+/// assert_eq!(batch.column(1).as_primitive::<Float64Type>().value(0), 1.0);
+///
+/// // Dispatched on metadata, and decoded by the reader's own `Utf8` decoder
+/// assert_eq!(batch.column(3).as_string::<i32>().value(0), "HI");
+/// ```
+///
+/// [`EncoderFactory`]: crate::EncoderFactory
+pub trait DecoderFactory: std::fmt::Debug + Send + Sync {
+ /// Make a decoder for `field`, or `Ok(None)` to use the reader's default.
+ ///
+ /// Receives the [`FieldRef`] rather than just its [`DataType`] so decoder
+ /// selection can consider the field's metadata, e.g. to identify
[extension
+ /// types]. The root of a [`ReaderBuilder::new`] schema is presented as a
+ /// synthesized nameless `Struct` field.
+ ///
+ /// Use [`DecoderContext::make_decoder`] on `ctx` to build child decoders,
and
+ /// [`DecoderContext::make_builtin_decoder`] to build on the reader's own
decoder
+ /// for this field. Calling `make_decoder` with the field this was invoked
with
+ /// recurses back here and loops.
+ ///
+ /// `is_nullable` folds in ancestor nullability, so it may differ from
+ /// `field.is_nullable()` in either direction: a nullable struct widens its
+ /// children, while a run-end encoded array narrows its values.
+ ///
+ /// [extension types]:
https://arrow.apache.org/docs/format/Columnar.html#extension-types
+ fn make_default_decoder(
+ &self,
+ _ctx: &DecoderContext,
+ _field: &FieldRef,
+ _is_nullable: bool,
+ ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
+ Ok(None)
+ }
+}
+
+/// Validates the output of a [`DecoderFactory`] decoder before it reaches the
arrays
+/// built from it, some of which are constructed without further checks.
+struct CheckedDecoder {
+ inner: Box<dyn ArrayDecoder>,
+ data_type: DataType,
+}
+
+impl ArrayDecoder for CheckedDecoder {
+ fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError> {
+ let array = self.inner.decode(tape, pos)?;
+ if array.data_type() != &self.data_type {
+ return Err(ArrowError::JsonError(format!(
+ "custom decoder returned {} for a field of type {}",
+ array.data_type(),
+ self.data_type
+ )));
+ }
+ if array.len() != pos.len() {
+ return Err(ArrowError::JsonError(format!(
+ "custom decoder returned {} values for {} rows",
+ array.len(),
+ pos.len()
+ )));
+ }
+ Ok(array)
+ }
+}
+
/// Context for decoder creation, containing configuration.
///
/// This context is passed through the decoder creation process and contains
@@ -753,6 +963,8 @@ pub struct DecoderContext {
struct_mode: StructMode,
/// Whether to treat columns with incompatible types as missing (i.e. NULL)
ignore_type_conflicts: bool,
+ /// An optional hook for customizing decoding behavior
+ decoder_factory: Option<Arc<dyn DecoderFactory>>,
}
impl DecoderContext {
@@ -776,24 +988,61 @@ impl DecoderContext {
self.ignore_type_conflicts
}
+ /// Returns the optional hook for customizing decoding behavior
+ pub fn decoder_factory(&self) -> Option<&Arc<dyn DecoderFactory>> {
+ self.decoder_factory.as_ref()
+ }
+
/// Create a decoder for a type.
///
/// This is the standard way to create child decoders from within a decoder
/// implementation.
- fn make_decoder(
+ pub fn make_decoder(
&self,
- data_type: &DataType,
+ field: &FieldRef,
is_nullable: bool,
) -> Result<Box<dyn ArrayDecoder>, ArrowError> {
- make_decoder(self, data_type, is_nullable)
+ make_decoder(self, field, is_nullable)
+ }
+
+ /// Create the decoder the reader would use for `field`, ignoring any
+ /// [`DecoderFactory`].
+ ///
+ /// Lets a factory build on the reader's own decoder, including for the
field it was
+ /// asked about — which [`Self::make_decoder`] cannot do without looping.
+ pub fn make_builtin_decoder(
+ &self,
+ field: &FieldRef,
+ is_nullable: bool,
+ ) -> Result<Box<dyn ArrayDecoder>, ArrowError> {
+ make_builtin_decoder(self, field, is_nullable)
}
}
fn make_decoder(
ctx: &DecoderContext,
- data_type: &DataType,
+ field: &FieldRef,
is_nullable: bool,
) -> Result<Box<dyn ArrayDecoder>, ArrowError> {
+ if let Some(factory) = ctx.decoder_factory()
+ && let Some(decoder) = factory.make_default_decoder(ctx, field,
is_nullable)?
+ {
+ return Ok(Box::new(CheckedDecoder {
+ inner: decoder,
+ data_type: field.data_type().clone(),
+ }));
+ }
+
+ make_builtin_decoder(ctx, field, is_nullable)
+}
+
+fn make_builtin_decoder(
+ ctx: &DecoderContext,
+ field: &FieldRef,
+ is_nullable: bool,
+) -> Result<Box<dyn ArrayDecoder>, ArrowError> {
+ let data_type = field.data_type();
+
macro_rules! primitive_decoder {
($t:ty, $data_type:expr) => {
Ok(Box::new(PrimitiveArrayDecoder::<$t>::new(ctx, $data_type)))
@@ -898,6 +1147,7 @@ mod tests {
use serde_json::json;
use std::fs::File;
use std::io::{BufReader, Cursor, Seek};
+ use std::sync::Mutex;
use super::*;
@@ -3944,4 +4194,135 @@ mod tests {
assert_eq!(col.value(i), (i as i32) + 1);
}
}
+
+ /// Declining everything must be indistinguishable from no factory at all.
+ #[test]
+ fn test_decoder_factory_declining_everything_is_transparent() {
+ #[derive(Debug, Default)]
+ struct DeclineAll {
+ seen: Mutex<Vec<FieldRef>>,
+ }
+
+ impl DecoderFactory for DeclineAll {
+ fn make_default_decoder(
+ &self,
+ _ctx: &DecoderContext,
+ field: &FieldRef,
+ _is_nullable: bool,
+ ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
+ self.seen.lock().unwrap().push(field.clone());
+ Ok(None)
+ }
+ }
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Utf8, true),
+ Field::new_list("c", Field::new("item", DataType::Float64, true),
true),
+ ]));
+ let buf = r#"{"a": 1, "b": "x", "c": [1.5, 2.5]}
+ {"a": null, "c": []}
+ "#;
+
+ let read = |factory: Option<Arc<dyn DecoderFactory>>| {
+ let mut builder = ReaderBuilder::new(schema.clone());
+ if let Some(factory) = factory {
+ builder = builder.with_decoder_factory(factory);
+ }
+ builder
+ .build(Cursor::new(buf.as_bytes()))
+ .unwrap()
+ .next()
+ .unwrap()
+ .unwrap()
+ };
+
+ let factory = Arc::new(DeclineAll::default());
+ assert_eq!(read(None), read(Some(factory.clone())));
+
+ // Consulted for the root struct and every leaf, with names attached;
the
+ // root is the documented synthesized nameless struct
+ let seen = factory.seen.lock().unwrap();
+ assert!(
+ matches!(seen[0].data_type(), DataType::Struct(_)),
+ "{seen:?}"
+ );
+ assert_eq!(seen[0].name(), "", "root field should be nameless");
+
+ let by_name: Vec<(&str, &DataType)> = seen
+ .iter()
+ .map(|f| (f.name().as_str(), f.data_type()))
+ .collect();
+ for expected in [
+ ("a", &DataType::Int32),
+ ("b", &DataType::Utf8),
+ ("item", &DataType::Float64),
+ ] {
+ assert!(
+ by_name.contains(&expected),
+ "{expected:?} missing from {by_name:?}"
+ );
+ }
+ }
+
+ /// A custom decoder's output is not trusted: the arrays built from it are
+ /// constructed without further validation, so a mismatch must be caught
here.
+ #[test]
+ fn test_decoder_factory_output_is_validated() {
+ /// Returns either the wrong data type or the wrong number of rows
+ struct Bad {
+ wrong_type: bool,
+ }
+
+ impl ArrayDecoder for Bad {
+ fn decode(&mut self, _tape: &Tape<'_>, pos: &[u32]) ->
Result<ArrayRef, ArrowError> {
+ Ok(match self.wrong_type {
+ true => Arc::new(StringViewArray::from(vec!["x";
pos.len()])) as ArrayRef,
+ false => Arc::new(StringArray::from(Vec::<&str>::new())),
+ })
+ }
+ }
+
+ #[derive(Debug)]
+ struct BadFactory {
+ wrong_type: bool,
+ }
+
+ impl DecoderFactory for BadFactory {
+ fn make_default_decoder(
+ &self,
+ _ctx: &DecoderContext,
+ field: &FieldRef,
+ _is_nullable: bool,
+ ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
+ match field.data_type() {
+ DataType::Utf8 => Ok(Some(Box::new(Bad {
+ wrong_type: self.wrong_type,
+ }))),
+ _ => Ok(None),
+ }
+ }
+ }
+
+ let read = |wrong_type: bool| {
+ let schema = Arc::new(Schema::new(vec![Field::new("s",
DataType::Utf8, true)]));
+ ReaderBuilder::new(schema)
+ .with_decoder_factory(Arc::new(BadFactory { wrong_type }))
+ .build(Cursor::new(br#"{"s": "hello"}"#.as_slice()))
+ .unwrap()
+ .next()
+ .unwrap()
+ .unwrap_err()
+ .to_string()
+ };
+
+ let err = read(true);
+ assert!(
+ err.contains("returned Utf8View for a field of type Utf8"),
+ "{err}"
+ );
+
+ let err = read(false);
+ assert!(err.contains("returned 0 values for 1 rows"), "{err}");
+ }
}
diff --git a/arrow-json/src/reader/run_end_array.rs
b/arrow-json/src/reader/run_end_array.rs
index df9952f100..4c4450a7c2 100644
--- a/arrow-json/src/reader/run_end_array.rs
+++ b/arrow-json/src/reader/run_end_array.rs
@@ -47,7 +47,7 @@ impl<R: RunEndIndexType> RunEndEncodedArrayDecoder<R> {
unreachable!()
};
let values_nullable = values_field.is_nullable() && is_nullable;
- let decoder = ctx.make_decoder(values_field.data_type(),
values_nullable)?;
+ let decoder = ctx.make_decoder(values_field, values_nullable)?;
Ok(Self {
data_type: data_type.clone(),
diff --git a/arrow-json/src/reader/struct_array.rs
b/arrow-json/src/reader/struct_array.rs
index 6de81a6c8d..2149d8ba04 100644
--- a/arrow-json/src/reader/struct_array.rs
+++ b/arrow-json/src/reader/struct_array.rs
@@ -95,7 +95,7 @@ impl StructArrayDecoder {
// StructArrayDecoder::decode verifies that if the child is
not nullable
// it doesn't contain any nulls not masked by its parent
let nullable = f.is_nullable() || is_nullable;
- ctx.make_decoder(f.data_type(), nullable)
+ ctx.make_decoder(f, nullable)
})
.collect::<Result<Vec<_>, ArrowError>>()?;
diff --git a/arrow-json/src/reader/tape.rs b/arrow-json/src/reader/tape.rs
index d493f6cb70..42780f0f5d 100644
--- a/arrow-json/src/reader/tape.rs
+++ b/arrow-json/src/reader/tape.rs
@@ -29,8 +29,16 @@ use std::fmt::Write;
/// Uses `u32` for offsets to ensure `TapeElement` is 64-bits. A future
/// iteration may increase this to a custom `u56` type.
///
+/// Numbers take more than one form, and matches must handle all of them.
Parsing JSON
+/// text always yields [`Self::Number`], holding the value textually (read via
+/// [`Tape::get_string`]). Serializing Rust values (see
+/// [`Decoder::serialize`](super::Decoder::serialize)) yields [`Self::I32`],
+/// [`Self::I64`], [`Self::F32`] or [`Self::F64`] — 64-bit values spanning two
+/// elements, high bits first — or [`Self::Number`] for integers exceeding
`i64`.
+///
/// [simdjson]: https://github.com/simdjson/simdjson/blob/master/doc/tape.md
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[non_exhaustive]
pub enum TapeElement {
/// The start of an object, i.e. `{`
///
@@ -91,6 +99,9 @@ pub enum TapeElement {
///
/// This approach to decoding JSON is inspired by [simdjson]
///
+/// String data is copied into the tape with escapes resolved, so
[`Tape::get_string`]
+/// borrows from the tape, not the input. A `Tape` is read, never constructed.
+///
/// [simdjson]: https://github.com/simdjson/simdjson/blob/master/doc/tape.md
#[derive(Debug)]
pub struct Tape<'a> {
