This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 486867ff2e Document how to customize JSON decoding via
`DecoderFactory` (#10897)
486867ff2e is described below
commit 486867ff2e8676c14cca235e0751b7924eca700d
Author: Andrew Lamb <[email protected]>
AuthorDate: Thu Sep 24 01:57:52 2026 -0400
Document how to customize JSON decoding via `DecoderFactory` (#10897)
# Which issue does this PR close?
N/A -- documentation only.
- Follow on to #10670
# Rationale for this change
- #10670 adds the ability to customize JSON decoding via
[`DecoderFactory`], but the docs don't mention it, so users would only
find the hook by stumbling on `ReaderBuilder::with_decoder_factory`.
[`DecoderFactory`]: https://github.com/apache/arrow-rs/pull/10670
# What changes are included in this PR?
This adds the matching pointers for *decoding* that #10741 added for
*encoding*, as anticipated in that PR's description.
I also cleaned up some other docs while I was reviewing the original PR
# Are these changes tested?
Covered by existing doc tests and CI rustdoc link checking.
# Are there any user-facing changes?
Documentation only.
---
arrow-json/src/lib.rs | 4 +-
arrow-json/src/reader/mod.rs | 141 +++++++++++++++++++++++++++------------
arrow-json/src/writer/encoder.rs | 17 ++++-
3 files changed, 119 insertions(+), 43 deletions(-)
diff --git a/arrow-json/src/lib.rs b/arrow-json/src/lib.rs
index 9e0254f268..322c903a9f 100644
--- a/arrow-json/src/lib.rs
+++ b/arrow-json/src/lib.rs
@@ -29,11 +29,13 @@
//! Note that `Base16` only has 50% space efficiency (i.e., the encoded data
is twice as large
//! as the original). If that is an issue, there are two alternatives:
//!
-//! 1. Provide a custom encoder. See the [Customizing the encoder] section of
the writer documentation.
+//! 1. Provide a custom encoder and/or decoder. See the [Customizing the
encoder] section of the
+//! writer documentation and the [Customizing the decoder] section of the
reader documentation.
//! 2. Convert binary data to/from a different encoding format such as
`Base64` before
//! writing / after reading, as shown in the following example.
//!
//! [Customizing the encoder]: writer#customizing-the-encoder
+//! [Customizing the decoder]: reader#customizing-the-decoder
//!
//! ## `Base64` Encoding Example
//!
diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs
index b1c110dd7a..8ac122d925 100644
--- a/arrow-json/src/reader/mod.rs
+++ b/arrow-json/src/reader/mod.rs
@@ -49,11 +49,11 @@
//!
//! # Async Usage
//!
-//! The lower-level [`Decoder`] can be integrated with various forms of async
data streams,
-//! and is designed to be agnostic to the various different kinds of async IO
primitives found
+//! The lower-level [`Decoder`] can be integrated with various forms of async
+//! data streams, and can be used with any of the async IO primitives found
//! within the Rust ecosystem.
//!
-//! For example, see below for how it can be used with an arbitrary `Stream`
of `Bytes`
+//! For example, it can be used with an arbitrary `Stream` of `Bytes` as shown
below
//!
//! ```
//! # use std::task::{Poll, ready};
@@ -132,6 +132,14 @@
//! }
//! ```
//!
+//! # Customizing the decoder
+//!
+//! The decoding of each data type can be customized using
+//! [`ReaderBuilder::with_decoder_factory`]. For example, you can override the
+//! default decoding of binary data from hex strings to decode from an array of
+//! integers instead, or provide decoders for types with no built-in decoding.
+//! See the example on [`DecoderFactory`].
+//!
use std::io::BufRead;
use std::sync::Arc;
@@ -164,7 +172,8 @@ use crate::reader::tape::{TapeDecoder, TapeDecoderOptions};
use crate::reader::timestamp_array::TimestampArrayDecoder;
pub use schema::*;
-// `mod tape` stays private so `TapeDecoder` does not become public API
+// `mod tape` is private by design: exposing only these two types means
downstream can read a
+// `Tape` but not construct one, keeping `TapeDecoder` an implementation
detail.
pub use tape::{Tape, TapeElement};
pub use value_iter::ValueIter;
@@ -766,20 +775,20 @@ pub trait ArrayDecoder: Send {
/// handle. The reader-side counterpart of [`EncoderFactory`]; register an
/// implementation with [`ReaderBuilder::with_decoder_factory`].
///
-/// # Examples
+/// # Example: Decode `Binary` from a JSON array of integers
///
-/// Decodes `Binary` from a JSON array of integers rather than the default hex
string.
+/// This example shows how to decode a `Binary` column from a JSON array of
+/// integers, e.g. `[104, 105]` for the bytes `b"hi"`, as created by the
+/// example in [`EncoderFactory`].
///
/// ```
-/// use std::sync::Arc;
-/// use arrow_array::{Array, ArrayRef, BinaryArray};
-/// use arrow_array::types::Float64Type;
-/// use arrow_array::cast::AsArray;
-/// use arrow_json::reader::{ArrayDecoder, DecoderContext, DecoderFactory,
Tape, TapeElement};
-/// use arrow_json::ReaderBuilder;
-/// use arrow_schema::{ArrowError, DataType, Field, FieldRef, Fields, Schema};
-/// use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
-/// use arrow_array::StringArray;
+/// # use std::sync::Arc;
+/// # use arrow_array::{Array, ArrayRef, BinaryArray};
+/// # use arrow_array::types::Float64Type;
+/// # use arrow_array::cast::AsArray;
+/// # use arrow_json::reader::{ArrayDecoder, DecoderContext, DecoderFactory,
Tape, TapeElement};
+/// # use arrow_json::ReaderBuilder;
+/// # use arrow_schema::{ArrowError, DataType, Field, FieldRef, Fields,
Schema};
///
/// /// Decodes `[104, 105]` into the bytes `b"hi"`
/// struct IntArrayBinaryDecoder;
@@ -816,19 +825,7 @@ pub trait ArrayDecoder: Send {
/// }
/// }
///
-/// /// Upper-cases whatever the reader's own decoder produced
-/// struct ShoutDecoder(Box<dyn ArrayDecoder>);
-///
-/// impl ArrayDecoder for ShoutDecoder {
-/// fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError> {
-/// let inner = self.0.decode(tape, pos)?;
-/// let values = inner.as_string::<i32>();
-/// Ok(Arc::new(StringArray::from_iter(
-/// values.iter().map(|v| v.map(str::to_uppercase)),
-/// )))
-/// }
-/// }
-///
+/// /// A factory that overrides the reader's default decoder for `Binary` to
decode from an array of integers
/// #[derive(Debug)]
/// struct IntArrayBinaryDecoderFactory;
///
@@ -839,15 +836,6 @@ pub trait ArrayDecoder: Send {
/// field: &FieldRef,
/// is_nullable: bool,
/// ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
-/// // Selection can key off metadata, e.g. to recognise an extension
type, and
-/// // build on the reader's own decoder for the very same field
-/// if
field.metadata().get(EXTENSION_TYPE_NAME_KEY).map(String::as_str)
-/// == Some("apache.shout")
-/// {
-/// let inner = ctx.make_builtin_decoder(field, is_nullable)?;
-/// return Ok(Some(Box::new(ShoutDecoder(inner))));
-/// }
-///
/// match field.data_type() {
/// DataType::Binary => Ok(Some(Box::new(IntArrayBinaryDecoder))),
/// // Returning `None` uses the reader's default decoder
@@ -861,11 +849,9 @@ pub trait ArrayDecoder: Send {
/// Field::new("bytes", DataType::Binary, true),
/// Field::new("float", DataType::Float64, true),
/// Field::new("nested", DataType::Struct(nested), true),
-/// Field::new("shout", DataType::Utf8, true)
-/// .with_metadata([(EXTENSION_TYPE_NAME_KEY, "apache.shout")]),
/// ]));
///
-/// let json = r#"{"bytes": [104, 105], "float": 1.0, "nested": {"inner":
[104, 105]}, "shout": "hi"}
+/// let json = r#"{"bytes": [104, 105], "float": 1.0, "nested": {"inner":
[104, 105]}}
/// {"float": 2.3}
/// {"bytes": [98], "nested": {"inner": [98]}}
/// "#;
@@ -888,12 +874,85 @@ pub trait ArrayDecoder: Send {
/// let inner = batch.column(2).as_struct().column(0).as_binary::<i32>();
/// assert_eq!(inner.value(0), b"hi");
/// assert_eq!(batch.column(1).as_primitive::<Float64Type>().value(0), 1.0);
+/// ```
+///
+/// # Example: Decode an [extension type] from a JSON string
+///
+/// The choice of decoder can also depend on field metadata rather than just
+/// the data type, for example to recognize an [extension type]. The reader's
+/// own decoder for the field can be invoked with
+/// [`DecoderContext::make_builtin_decoder`].
+///
+/// This example upper-cases the strings produced by the builtin `Utf8` decoder
+/// for a fictional extension type `apache.shout`.
+///
+/// ```
+/// # use std::sync::Arc;
+/// # use arrow_array::{ArrayRef, StringArray};
+/// # use arrow_array::cast::AsArray;
+/// # use arrow_json::reader::{ArrayDecoder, DecoderContext, DecoderFactory,
Tape};
+/// # use arrow_json::ReaderBuilder;
+/// # use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema};
+/// # use arrow_schema::extension::EXTENSION_TYPE_NAME_KEY;
+///
+/// /// Upper-cases whatever the reader's own decoder produced
+/// struct ShoutDecoder(Box<dyn ArrayDecoder>);
+///
+/// impl ArrayDecoder for ShoutDecoder {
+/// fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result<ArrayRef,
ArrowError> {
+/// let inner = self.0.decode(tape, pos)?;
+/// let values = inner.as_string::<i32>();
+/// Ok(Arc::new(StringArray::from_iter(
+/// values.iter().map(|v| v.map(str::to_uppercase)),
+/// )))
+/// }
+/// }
+///
+/// /// A factory that recognizes the `apache.shout` extension type and
upper-cases its strings
+/// #[derive(Debug)]
+/// struct ShoutDecoderFactory;
+///
+/// impl DecoderFactory for ShoutDecoderFactory {
+/// fn make_default_decoder(
+/// &self,
+/// ctx: &DecoderContext,
+/// field: &FieldRef,
+/// is_nullable: bool,
+/// ) -> Result<Option<Box<dyn ArrayDecoder>>, ArrowError> {
+/// if
field.metadata().get(EXTENSION_TYPE_NAME_KEY).map(String::as_str)
+/// == Some("apache.shout")
+/// {
+/// let inner = ctx.make_builtin_decoder(field, is_nullable)?;
+/// return Ok(Some(Box::new(ShoutDecoder(inner))));
+/// }
+/// Ok(None)
+/// }
+/// }
+///
+/// let schema = Arc::new(Schema::new(vec![
+/// Field::new("shout", DataType::Utf8, true)
+/// .with_metadata([(EXTENSION_TYPE_NAME_KEY, "apache.shout")]),
+/// Field::new("plain", DataType::Utf8, true),
+/// ]));
+///
+/// let json = r#"{"shout": "hi", "plain": "hi"}"#;
+///
+/// let batch = ReaderBuilder::new(schema)
+/// .with_decoder_factory(Arc::new(ShoutDecoderFactory))
+/// .build(json.as_bytes())
+/// .unwrap()
+/// .next()
+/// .unwrap()
+/// .unwrap();
///
/// // Dispatched on metadata, and decoded by the reader's own `Utf8` decoder
-/// assert_eq!(batch.column(3).as_string::<i32>().value(0), "HI");
+/// assert_eq!(batch.column(0).as_string::<i32>().value(0), "HI");
+/// // The factory declines fields without the extension type, so they are
decoded as usual
+/// assert_eq!(batch.column(1).as_string::<i32>().value(0), "hi");
/// ```
///
/// [`EncoderFactory`]: crate::EncoderFactory
+/// [extension type]:
https://arrow.apache.org/docs/format/Columnar.html#extension-types
pub trait DecoderFactory: std::fmt::Debug + Send + Sync {
/// Make a decoder for `field`, or `Ok(None)` to use the reader's default.
///
diff --git a/arrow-json/src/writer/encoder.rs b/arrow-json/src/writer/encoder.rs
index 5a6c774e29..696af0b1ab 100644
--- a/arrow-json/src/writer/encoder.rs
+++ b/arrow-json/src/writer/encoder.rs
@@ -149,7 +149,14 @@ impl EncoderOptions {
///
/// [`WriterBuilder::with_encoder_factory`]:
crate::writer::WriterBuilder::with_encoder_factory
///
-/// # Examples
+/// # Example: Encode a `BinaryArray` as an array of integers
+///
+/// This example encodes each `BinaryArray` value as an array of integers. For
+/// example, the bytes `b"abc"` would be encoded as `[97, 98, 99]`. See the
+/// example on [`DecoderFactory`] for how to decode this back into a
+/// `BinaryArray`.
+///
+/// [`DecoderFactory`]: crate::DecoderFactory
///
/// ```
/// use std::io::Write;
@@ -163,6 +170,8 @@ impl EncoderOptions {
/// use serde_json::json;
/// use serde_json::Value;
///
+/// /// Encoder for `BinaryArray` that encodes the bytes as an array of
integers.
+/// /// For example, the bytes `b"abc"` would be encoded as `[97, 98, 99]`.
/// struct IntArrayBinaryEncoder<B> {
/// array: B,
/// }
@@ -184,6 +193,7 @@ impl EncoderOptions {
/// }
/// }
///
+/// /// Factory that creates an `IntArrayBinaryEncoder` for `BinaryArray`
types.
/// #[derive(Debug)]
/// struct IntArrayBinaryEncoderFactory;
///
@@ -207,6 +217,9 @@ impl EncoderOptions {
/// }
/// }
///
+/// // The input has two columns:
+/// // bytes: [b"a", null, b"b"]
+/// // float: [1.0, 2.3, null]
/// let binary_array = BinaryArray::from_iter([Some(b"a".as_slice()), None,
Some(b"b".as_slice())]);
/// let float_array = Float64Array::from(vec![Some(1.0), Some(2.3), None]);
/// let fields = vec