This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new ebd3ecbcd8 Optimize PLAIN parquet string column to dictionary array
(#10614)
ebd3ecbcd8 is described below
commit ebd3ecbcd837398bd4f821b5a96d4a4dc9d74662
Author: RIchard Baah <[email protected]>
AuthorDate: Sat Aug 22 00:36:57 2026 -0400
Optimize PLAIN parquet string column to dictionary array (#10614)
# Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.
-->
- Closes #10590
- builds on top of #10591
- works towards https://github.com/apache/datafusion/issues/24111
# Rationale for this change
- see #10590
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
# What changes are included in this PR?
Optimizes plain-encoded string/binary columns read into Dictionary Arrow
arrays by eliminating the intermediate StringArray materialization.
Previously, all values were decoded into a flat array first, then
iterated again to build the dictionary. Now, hashes are computed upfront
in a single batch pass and used to deduplicate values directly into the
dictionary output buffers, reducing the work to a single pass over the
data. The hash scratch buffer is also reused across batches, so
allocation overhead amortizes as batch sizes grow.
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
# Are these changes tested?
yes existing test cover this functionality. nothing logically has
changed
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
If this PR claims a performance improvement, please include evidence
such as benchmark results.
-->
# Are there any user-facing changes?
~~yes, `DictionaryBuffer::into_array()` now takes in scratch vector.
this is a breaking change~~
EDIT: no breaking changes, `DictionaryBuffer` isn't part of public API
---------
Co-authored-by: Jeffrey Vo <[email protected]>
---
.../arrow/array_reader/byte_array_dictionary.rs | 35 +-
parquet/src/arrow/buffer/dictionary_buffer.rs | 367 ++++++++++++++++++---
2 files changed, 342 insertions(+), 60 deletions(-)
diff --git a/parquet/src/arrow/array_reader/byte_array_dictionary.rs
b/parquet/src/arrow/array_reader/byte_array_dictionary.rs
index f4d84503bb..898df57d7e 100644
--- a/parquet/src/arrow/array_reader/byte_array_dictionary.rs
+++ b/parquet/src/arrow/array_reader/byte_array_dictionary.rs
@@ -19,7 +19,7 @@ use std::any::Any;
use std::marker::PhantomData;
use arrow_array::{Array, ArrayRef, OffsetSizeTrait, new_empty_array};
-use arrow_buffer::ArrowNativeType;
+use arrow_buffer::{ArrowNativeType, MutableBuffer};
use arrow_schema::DataType as ArrowType;
use bytes::Bytes;
@@ -128,6 +128,8 @@ struct ByteArrayDictionaryReader<K: ArrowNativeType, V:
OffsetSizeTrait> {
def_levels_buffer: Option<Vec<i16>>,
rep_levels_buffer: Option<Vec<i16>>,
record_reader: GenericRecordReader<DictionaryBuffer<K, V>,
DictionaryDecoder<K, V>>,
+ /// Reusable scratch space for hashing byte slices when building
dictionaries from plain-encoded values.
+ hash_scratch: MutableBuffer,
}
impl<K, V> ByteArrayDictionaryReader<K, V>
@@ -146,6 +148,7 @@ where
def_levels_buffer: None,
rep_levels_buffer: None,
record_reader,
+ hash_scratch: MutableBuffer::new(0),
}
}
}
@@ -181,7 +184,7 @@ where
let buffer = self.record_reader.consume_record_data();
let null_buffer = self.record_reader.consume_compact_bitmap();
- let array = buffer.into_array(null_buffer, &self.data_type)?;
+ let array = buffer.into_array(null_buffer, &self.data_type, &mut
self.hash_scratch)?;
self.record_reader.reset();
Ok(array)
@@ -459,7 +462,9 @@ mod tests {
assert!(matches!(output, DictionaryBuffer::Dict { .. }));
- let array = output.into_array(Some(valid_buffer), &data_type).unwrap();
+ let array = output
+ .into_array(Some(valid_buffer), &data_type, &mut
MutableBuffer::new(0))
+ .unwrap();
assert_eq!(array.data_type(), &data_type);
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -530,7 +535,9 @@ mod tests {
assert!(matches!(output, DictionaryBuffer::Dict { .. }));
- let array = output.into_array(Some(valid_buffer), &data_type).unwrap();
+ let array = output
+ .into_array(Some(valid_buffer), &data_type, &mut
MutableBuffer::new(0))
+ .unwrap();
assert_eq!(array.data_type(), &data_type);
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -565,7 +572,9 @@ mod tests {
decoder.set_data(encoding, page, 4, Some(4)).unwrap();
assert_eq!(decoder.read(&mut output, 1024).unwrap(), 4);
}
- let array = output.into_array(None, &data_type).unwrap();
+ let array = output
+ .into_array(None, &data_type, &mut MutableBuffer::new(0))
+ .unwrap();
assert_eq!(array.data_type(), &data_type);
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -609,7 +618,9 @@ mod tests {
decoder.skip_values(2).expect("skipping two values");
assert_eq!(decoder.read(&mut output, 1024).unwrap(), 2);
}
- let array = output.into_array(None, &data_type).unwrap();
+ let array = output
+ .into_array(None, &data_type, &mut MutableBuffer::new(0))
+ .unwrap();
assert_eq!(array.data_type(), &data_type);
let array = cast(&array, &ArrowType::Utf8).unwrap();
@@ -672,7 +683,11 @@ mod tests {
output.pad_nulls(0, 0, 8, &[0]).unwrap();
let array = output
- .into_array(Some(Buffer::from(&[0])), &data_type)
+ .into_array(
+ Some(Buffer::from(&[0])),
+ &data_type,
+ &mut MutableBuffer::new(0),
+ )
.unwrap();
assert_eq!(array.len(), 8);
@@ -687,7 +702,11 @@ mod tests {
output.pad_nulls(0, 0, 8, &[0]).unwrap();
let array = output
- .into_array(Some(Buffer::from(&[0])), &data_type)
+ .into_array(
+ Some(Buffer::from(&[0])),
+ &data_type,
+ &mut MutableBuffer::new(0),
+ )
.unwrap();
assert_eq!(array.len(), 8);
diff --git a/parquet/src/arrow/buffer/dictionary_buffer.rs
b/parquet/src/arrow/buffer/dictionary_buffer.rs
index 1272b88ec1..c2f92bf4c5 100644
--- a/parquet/src/arrow/buffer/dictionary_buffer.rs
+++ b/parquet/src/arrow/buffer/dictionary_buffer.rs
@@ -18,17 +18,21 @@
use crate::arrow::buffer::offset_buffer::OffsetBuffer;
use crate::arrow::record_reader::buffer::ValuesBuffer;
use crate::errors::{ParquetError, Result};
-use arrow_array::{Array, GenericByteArray, downcast_integer};
+use ahash::RandomState;
+use arrow_array::{Array, DictionaryArray, downcast_integer};
use arrow_array::{
- ArrayRef, FixedSizeBinaryArray, OffsetSizeTrait,
- builder::{FixedSizeBinaryDictionaryBuilder, GenericByteDictionaryBuilder},
- cast::AsArray,
- make_array,
- types::{ArrowDictionaryKeyType, ByteArrayType},
+ ArrayRef, FixedSizeBinaryArray, OffsetSizeTrait, cast::AsArray, make_array,
+ types::ArrowDictionaryKeyType,
};
-use arrow_buffer::{ArrowNativeType, Buffer};
+use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
use arrow_data::ArrayDataBuilder;
use arrow_schema::DataType as ArrowType;
+use hashbrown::HashMap as HbHashMap;
+use hashbrown::hash_map::Entry;
+use std::hash::{BuildHasher, Hasher};
+use std::mem::size_of;
+use std::ptr::write_unaligned;
+use std::slice::{from_raw_parts, from_raw_parts_mut};
use std::sync::Arc;
/// An array of variable length byte arrays that are potentially dictionary
encoded
@@ -127,6 +131,7 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait>
DictionaryBuffer<K, V> {
self,
null_buffer: Option<Buffer>,
data_type: &ArrowType,
+ hash_scratch: &mut MutableBuffer,
) -> Result<ArrayRef> {
assert!(matches!(data_type, ArrowType::Dictionary(_, _)));
@@ -186,8 +191,10 @@ impl<K: ArrowNativeType + Ord, V: OffsetSizeTrait>
DictionaryBuffer<K, V> {
_ => unreachable!(),
};
- let array = values.into_array(null_buffer, value_type);
- pack_values(key_type, &array)
+ hash_byte_slices(&values.offsets, &values.values,
hash_scratch);
+ let hashes = hashes_as_u64(hash_scratch);
+
+ pack_values_from_offsets(key_type, &value_type, &values,
hashes, null_buffer)
}
}
}
@@ -226,58 +233,190 @@ impl<K: ArrowNativeType, V: OffsetSizeTrait>
ValuesBuffer for DictionaryBuffer<K
}
}
-macro_rules! dict_helper {
- ($k:ty, $array:ident) => {
- match $array.data_type() {
- ArrowType::Utf8 => pack_values_impl::<$k,
_>($array.as_string::<i32>()),
- ArrowType::LargeUtf8 => pack_values_impl::<$k,
_>($array.as_string::<i64>()),
- ArrowType::Binary => pack_values_impl::<$k,
_>($array.as_binary::<i32>()),
- ArrowType::LargeBinary => pack_values_impl::<$k,
_>($array.as_binary::<i64>()),
- ArrowType::FixedSizeBinary(_) => {
- pack_fixed_values_impl::<$k>($array.as_fixed_size_binary())
- }
- _ => unreachable!(),
- }
+macro_rules! offsets_dict_helper {
+ ($k:ty, $key_type:ident, $value_type:ident, $values:ident, $hashes:ident,
$null_buffer:ident) => {
+ pack_values_from_offsets_impl::<$k, _>(
+ $values,
+ $hashes,
+ $null_buffer,
+ $key_type,
+ $value_type,
+ )
};
}
-fn pack_values(key_type: &ArrowType, values: &ArrayRef) -> Result<ArrayRef> {
+fn pack_values_from_offsets<V: OffsetSizeTrait>(
+ key_type: &ArrowType,
+ value_type: &ArrowType,
+ values: &OffsetBuffer<V>,
+ hashes: &[u64],
+ null_buffer: Option<Buffer>,
+) -> Result<ArrayRef> {
downcast_integer! {
- key_type => (dict_helper, values),
- _ => unreachable!(),
+ key_type => (offsets_dict_helper, key_type, value_type, values,
hashes, null_buffer),
+ _ => unreachable!(),
}
}
-fn pack_values_impl<K: ArrowDictionaryKeyType, T: ByteArrayType>(
- array: &GenericByteArray<T>,
-) -> Result<ArrayRef> {
- let mut builder = GenericByteDictionaryBuilder::<K,
T>::with_capacity(array.len(), 1024, 1024);
- for x in array {
- match x {
- Some(x) => builder.append_value(x),
- None => builder.append_null(),
- }
+// Avoids double-hashing: keys are already high-quality u64 hashes from ahash,
+// so we pass them through directly rather than re-hashing inside the HashMap.
+struct PassthroughHasher(u64);
+impl std::hash::Hasher for PassthroughHasher {
+ fn finish(&self) -> u64 {
+ self.0
+ }
+ fn write(&mut self, _: &[u8]) {
+ unreachable!()
+ }
+ fn write_u64(&mut self, value: u64) {
+ self.0 = value;
+ }
+}
+#[derive(Default)]
+struct BuildPassthroughHasher;
+impl std::hash::BuildHasher for BuildPassthroughHasher {
+ type Hasher = PassthroughHasher;
+ fn build_hasher(&self) -> PassthroughHasher {
+ PassthroughHasher(0)
}
- let raw = builder.finish();
- Ok(Arc::new(raw))
}
-fn pack_fixed_values_impl<K: ArrowDictionaryKeyType>(
- array: &FixedSizeBinaryArray,
+/// Builds a [`DictionaryArray`] directly from a flat [`OffsetBuffer`] using
pre-computed
+/// hashes to deduplicate values in a single pass, avoiding the intermediate
StringArray
+/// materialization
+fn pack_values_from_offsets_impl<K: ArrowDictionaryKeyType, V:
OffsetSizeTrait>(
+ offset_buffer: &OffsetBuffer<V>,
+ hashes: &[u64],
+ null_buffer: Option<Buffer>,
+ key_type: &ArrowType,
+ value_type: &ArrowType,
) -> Result<ArrayRef> {
- let mut builder = FixedSizeBinaryDictionaryBuilder::<K>::with_capacity(
- array.len(),
- 1024,
- array.value_length(),
- );
- for x in array {
- match x {
- Some(x) => builder.append_value(x),
- None => builder.append_null(),
- }
+ let dict_type = ArrowType::Dictionary(Box::new(key_type.clone()),
Box::new(value_type.clone()));
+ let num_values = offset_buffer.len();
+
+ let mut keys: Vec<K::Native> = Vec::with_capacity(num_values);
+ let mut unique_offsets: Vec<V> = Vec::with_capacity(num_values + 1);
+ unique_offsets.push(V::default());
+ let mut unique_bytes: Vec<u8> =
Vec::with_capacity(offset_buffer.values.len());
+
+ let mut dedup: HbHashMap<u64, (usize, usize), BuildPassthroughHasher> =
+ HbHashMap::with_capacity_and_hasher(num_values,
BuildPassthroughHasher);
+
+ // Tracks colliding values (same hash, different bytes) so repeated
occurrences
+ // are deduplicated rather than inserted as new entries. Empty in the
common case.
+ let mut collision_overflow: Vec<(u64, usize, usize)> = Vec::new();
+
+ for (input_idx, &hash) in hashes.iter().enumerate() {
+ let byte_start = offset_buffer.offsets[input_idx].as_usize();
+ let byte_end = offset_buffer.offsets[input_idx + 1].as_usize();
+ let bytes = &offset_buffer.values[byte_start..byte_end];
+
+ let output_idx = match dedup.entry(hash) {
+ Entry::Occupied(entry) => {
+ let (first_input_idx, existing_output_idx) = *entry.get();
+ let first_start =
offset_buffer.offsets[first_input_idx].as_usize();
+ let first_end = offset_buffer.offsets[first_input_idx +
1].as_usize();
+ if &offset_buffer.values[first_start..first_end] == bytes {
+ existing_output_idx
+ } else {
+ // True hash collision — check overflow list before
inserting.
+ let existing = collision_overflow
+ .iter()
+ .find(|&&(entry_hash, collision_input_idx, _)| {
+ if entry_hash != hash {
+ return false;
+ }
+ let collision_start =
+
offset_buffer.offsets[collision_input_idx].as_usize();
+ let collision_end =
+ offset_buffer.offsets[collision_input_idx +
1].as_usize();
+
&offset_buffer.values[collision_start..collision_end] == bytes
+ })
+ .map(|&(_, _, collision_output_idx)|
collision_output_idx);
+
+ match existing {
+ Some(collision_output_idx) => collision_output_idx,
+ None => {
+ let new_output_idx = unique_offsets.len() - 1;
+ unique_bytes.extend_from_slice(bytes);
+ let new_end =
V::from_usize(unique_bytes.len()).ok_or_else(|| {
+ general_err!("offset overflow building
dictionary")
+ })?;
+ unique_offsets.push(new_end);
+ collision_overflow.push((hash, input_idx,
new_output_idx));
+ new_output_idx
+ }
+ }
+ }
+ }
+ Entry::Vacant(entry) => {
+ let output_idx = unique_offsets.len() - 1;
+ unique_bytes.extend_from_slice(bytes);
+ let new_end = V::from_usize(unique_bytes.len())
+ .ok_or_else(|| general_err!("offset overflow building
dictionary"))?;
+ unique_offsets.push(new_end);
+ entry.insert((input_idx, output_idx));
+ output_idx
+ }
+ };
+
+ let key = K::Native::from_usize(output_idx)
+ .ok_or_else(|| general_err!("dictionary key overflow"))?;
+ keys.push(key);
+ }
+
+ let num_unique = unique_offsets.len() - 1;
+
+ // SAFETY: buffers are constructed directly from typed Vecs above; offsets
are
+ // monotonically non-decreasing and bounded by unique_bytes.len(), and all
+ // key values are within 0..num_unique, so the invariants Arrow requires
hold.
+ let value_data = unsafe {
+ arrow_data::ArrayData::builder(value_type.clone())
+ .len(num_unique)
+ .add_buffer(Buffer::from_vec(unique_offsets))
+ .add_buffer(Buffer::from_vec(unique_bytes))
+ .build_unchecked()
+ };
+
+ // SAFETY: keys are within 0..num_unique and value_data is valid.
+ let dict_array: DictionaryArray<K> = unsafe {
+ arrow_data::ArrayData::builder(dict_type)
+ .len(keys.len())
+ .add_buffer(Buffer::from_vec(keys))
+ .add_child_data(value_data)
+ .null_bit_buffer(null_buffer)
+ .build_unchecked()
+ .into()
+ };
+
+ Ok(Arc::new(dict_array))
+}
+
+fn hash_byte_slices<I: ArrowNativeType>(offsets: &[I], values: &[u8], scratch:
&mut MutableBuffer) {
+ let count = offsets.len().saturating_sub(1);
+ scratch.clear();
+ scratch.resize(count * size_of::<u64>(), 0u8);
+
+ let state = RandomState::new();
+
+ // SAFETY: MutableBuffer is 64-byte aligned; scratch is sized to exactly
count * size_of::<u64>()
+ let hash_slots = unsafe {
from_raw_parts_mut(scratch.as_mut_ptr().cast::<u64>(), count) };
+
+ for idx in 0..count {
+ let start = offsets[idx].as_usize();
+ let end = offsets[idx + 1].as_usize();
+ let mut hasher = state.build_hasher();
+ hasher.write(&values[start..end]);
+ // SAFETY: idx is within 0..count
+ unsafe { write_unaligned(hash_slots.as_mut_ptr().add(idx),
hasher.finish()) };
}
- let raw = builder.finish();
- Ok(Arc::new(raw))
+}
+
+#[inline]
+fn hashes_as_u64(scratch: &[u8]) -> &[u64] {
+ let n = scratch.len() / size_of::<u64>();
+ // SAFETY: scratch was written as u64s by hash_byte_slices
+ unsafe { from_raw_parts(scratch.as_ptr().cast::<u64>(), n) }
}
#[cfg(test)]
@@ -285,6 +424,7 @@ mod tests {
use super::*;
use arrow::compute::cast;
use arrow_array::StringArray;
+ use arrow_array::types::*;
#[test]
fn test_dictionary_buffer() {
@@ -321,7 +461,9 @@ mod tests {
assert_eq!(buffer.len(), 13);
let split = std::mem::replace(&mut buffer,
DictionaryBuffer::with_capacity(0));
- let array = split.into_array(Some(null_buffer), &dict_type).unwrap();
+ let array = split
+ .into_array(Some(null_buffer), &dict_type, &mut
MutableBuffer::new(0))
+ .unwrap();
assert_eq!(array.data_type(), &dict_type);
let strings = cast(&array, &ArrowType::Utf8).unwrap();
@@ -355,7 +497,7 @@ mod tests {
.extend_from_slice(&[0, 1, 0, 1]);
let array = std::mem::replace(&mut buffer,
DictionaryBuffer::with_capacity(0))
- .into_array(None, &dict_type)
+ .into_array(None, &dict_type, &mut MutableBuffer::new(0))
.unwrap();
assert_eq!(array.data_type(), &dict_type);
@@ -386,7 +528,10 @@ mod tests {
let d = Arc::new(StringArray::from(vec!["", "f"])) as ArrayRef;
buffer.as_keys(&d).unwrap().extend_from_slice(&[0, 2, 0]);
- let err = buffer.into_array(None, &dict_type).unwrap_err().to_string();
+ let err = buffer
+ .into_array(None, &dict_type, &mut MutableBuffer::new(0))
+ .unwrap_err()
+ .to_string();
assert!(
err.contains("dictionary key beyond bounds of dictionary: 0..2"),
"{}",
@@ -404,4 +549,122 @@ mod tests {
err
);
}
+
+ /// A dictionary requested with LargeUtf8 values must come back with
LargeUtf8, not Utf8.
+ #[test]
+ fn test_values_path_large_utf8_type() {
+ let dict_type =
+ ArrowType::Dictionary(Box::new(ArrowType::Int32),
Box::new(ArrowType::LargeUtf8));
+ let mut buffer = DictionaryBuffer::<i32, i64>::with_capacity(0);
+ let values = buffer.spill_values().unwrap();
+ for s in ["foo", "bar", "foo"] {
+ values.try_push(s.as_bytes(), false).unwrap();
+ }
+
+ let array = buffer
+ .into_array(None, &dict_type, &mut MutableBuffer::new(0))
+ .unwrap();
+ let dict = array
+ .as_any()
+ .downcast_ref::<DictionaryArray<Int32Type>>()
+ .unwrap();
+
+ assert_eq!(dict.data_type(), &dict_type);
+ assert_eq!(dict.values().data_type(), &ArrowType::LargeUtf8);
+ }
+
+ /// 128 unique strings fill Int8 keys to capacity (last index = 127 =
i8::MAX).
+ /// A 129th unique string overflows and must return an error.
+ #[test]
+ fn test_int8_key_overflow_boundary() {
+ let dict_type = ArrowType::Dictionary(Box::new(ArrowType::Int8),
Box::new(ArrowType::Utf8));
+ let mut scratch = MutableBuffer::new(0);
+
+ {
+ let mut buffer = DictionaryBuffer::<i32, i32>::with_capacity(0);
+ let values = buffer.spill_values().unwrap();
+ for i in 0u32..128 {
+ values
+ .try_push(format!("val{i}").as_bytes(), false)
+ .unwrap();
+ }
+ buffer
+ .into_array(None, &dict_type, &mut scratch)
+ .expect("128 unique strings must fit: last index is 127 =
i8::MAX");
+ }
+
+ {
+ let mut buffer = DictionaryBuffer::<i32, i32>::with_capacity(0);
+ let values = buffer.spill_values().unwrap();
+ for i in 0u32..129 {
+ values
+ .try_push(format!("val{i}").as_bytes(), false)
+ .unwrap();
+ }
+ let err = buffer
+ .into_array(None, &dict_type, &mut scratch)
+ .unwrap_err()
+ .to_string();
+ assert!(
+ err.contains("dictionary key overflow"),
+ "expected 'dictionary key overflow', got: {err}"
+ );
+ }
+ }
+
+ /// A colliding string repeated after its initial collision must reuse its
key,
+ /// not be inserted again as a new dictionary entry.
+ #[test]
+ fn test_hash_collision_deduplication() {
+ let key_type = ArrowType::Int32;
+ let value_type = ArrowType::Utf8;
+
+ // "alpha", "beta", "beta" — all forced to share the same hash.
+ let mut ob = OffsetBuffer::<i32>::with_capacity(3);
+ ob.try_push(b"alpha", false).unwrap();
+ ob.try_push(b"beta", false).unwrap();
+ ob.try_push(b"beta", false).unwrap();
+ let hashes = [0xdeadbeef_u64; 3];
+
+ let array = pack_values_from_offsets_impl::<Int32Type, i32>(
+ &ob,
+ &hashes,
+ None,
+ &key_type,
+ &value_type,
+ )
+ .unwrap();
+
+ let dict = array
+ .as_any()
+ .downcast_ref::<DictionaryArray<Int32Type>>()
+ .unwrap();
+
+ assert_eq!(dict.values().len(), 2);
+ let keys: Vec<i32> = dict.keys().values().iter().copied().collect();
+ assert_eq!(keys, vec![0, 1, 1]);
+ }
+
+ /// A dictionary requested with Binary values must come back with Binary,
not Utf8.
+ #[test]
+ fn test_values_path_binary_type() {
+ let dict_type =
+ ArrowType::Dictionary(Box::new(ArrowType::Int32),
Box::new(ArrowType::Binary));
+ let mut buffer = DictionaryBuffer::<i32, i32>::with_capacity(0);
+ let values = buffer.spill_values().unwrap();
+ for s in [b"abc".as_ref(), b"\x00\xff", b"abc"] {
+ values.try_push(s, false).unwrap();
+ }
+
+ let array = buffer
+ .into_array(None, &dict_type, &mut MutableBuffer::new(0))
+ .unwrap();
+ let dict = array
+ .as_any()
+ .downcast_ref::<DictionaryArray<Int32Type>>()
+ .unwrap();
+
+ assert_eq!(dict.data_type(), &dict_type);
+ assert_eq!(dict.values().data_type(), &ArrowType::Binary);
+ }
}
