This is an automated email from the ASF dual-hosted git repository.
etseidl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new f271113651 bench(parquet): cover large dictionary values in
arrow_reader (#10691)
f271113651 is described below
commit f271113651537eccf42fd67edaae288c6461c2e3
Author: AarryaSaraf <[email protected]>
AuthorDate: Wed Aug 19 13:03:59 2026 -0700
bench(parquet): cover large dictionary values in arrow_reader (#10691)
# Which issue does this PR close?
None. This is benchmark coverage split out of #10690, so that the change
proposed there can be measured against a benchmark that already exists
on `main`.
It covers the shape reported in #10694.
# Rationale for this change
Every dictionary-encoded case in `arrow_reader` decodes ~20 byte values:
`build_dictionary_encoded_string_page_iterator` builds `"Dictionary
value {x}"`
at 1% unique. Two properties of that shape keep the dictionary gather
out of the
measurement entirely — values are small enough that per-key overhead
dominates
(RLE index decoding, the per-key bounds check), and the dictionary is a
couple of
KiB, so it stays cached for the whole decode.
Columns of large binary payloads invert both. A writer's dictionary size
limit is
checked lazily, so such a column is often dictionary encoded all the way
to the
end, with the dictionary page holding the entire column and each entry
referenced
exactly once. The gather then gets its data from a source far too large
to cache,
and it is the gather rather than the per-key work that dominates.
No benchmark in the crate covers that, so changes to
`OffsetBuffer::extend_from_dictionary` currently have nothing to be
measured
against.
# What changes are included in this PR?
One generator and one case in the existing `BinaryArray` group:
- `build_dictionary_encoded_large_value_page_iterator` — 64 KiB values,
all
distinct, 128 per page, mandatory (no NULLs), otherwise the same
row-group and
page geometry as the existing generators.
- `arrow_array_reader/BinaryArray/dictionary encoded, mandatory, no
NULLs, large values`
One iteration decodes 64 MiB of output from a 32 MiB dictionary.
Measured on
`main` (Apple M-series):
```
arrow_array_reader/BinaryArray/dictionary encoded, mandatory, no NULLs,
large values
time: [7.6857 ms 7.8982 ms 8.1303 ms]
```
# Are these changes tested?
This is a benchmark. It asserts its decoded value count on each run, as
the
surrounding cases do.
# Are there any user-facing changes?
No.
# AI disclosure
This benchmark was drafted with AI assistance and reviewed by me.
---
parquet/benches/arrow_reader.rs | 90 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/parquet/benches/arrow_reader.rs b/parquet/benches/arrow_reader.rs
index b86306ab8f..1d30076903 100644
--- a/parquet/benches/arrow_reader.rs
+++ b/parquet/benches/arrow_reader.rs
@@ -128,6 +128,18 @@ const BATCH_SIZE: usize = 8192;
const MAX_LIST_LEN: usize = 10;
const EXPECTED_VALUE_COUNT: usize = NUM_ROW_GROUPS * PAGES_PER_GROUP *
VALUES_PER_PAGE;
+// Params for the large dictionary value benchmark. Binary columns holding
large
+// payloads are commonly dictionary encoded in practice, because a writer's
+// dictionary size limit is checked lazily and so is never reached before the
+// column ends. Values there are distinct, which means the dictionary page
holds
+// the whole column and each entry is referenced exactly once: the gather reads
+// from a source far too large to stay cached, unlike the small-value cases
above
+// whose dictionary is a few KiB. Values are correspondingly larger and fewer
per
+// page, keeping one iteration to 64 MiB of output over a 32 MiB dictionary.
+const LARGE_VALUE_LEN: usize = 64 * 1024;
+const LARGE_VALUES_PER_PAGE: usize = 128;
+const EXPECTED_LARGE_VALUE_COUNT: usize = NUM_ROW_GROUPS * PAGES_PER_GROUP *
LARGE_VALUES_PER_PAGE;
+
pub fn seedable_rng() -> StdRng {
StdRng::seed_from_u64(42)
}
@@ -669,6 +681,67 @@ fn build_dictionary_encoded_string_page_iterator(
InMemoryPageIterator::new(pages)
}
+/// Builds pages of dictionary encoded values that are individually large and
all
+/// distinct, to cover the cost of gathering the dictionary values into the
output
+/// buffer. The small-value generator above is dominated by per-key overhead
+/// instead, and its dictionary is small enough to stay cached throughout.
+fn build_dictionary_encoded_large_value_page_iterator(
+ column_desc: ColumnDescPtr,
+) -> impl PageIterator + Clone {
+ use parquet::encoding::{DictEncoder, Encoder};
+ let max_def_level = column_desc.max_def_level();
+ let max_rep_level = column_desc.max_rep_level();
+ let rep_levels = vec![0; LARGE_VALUES_PER_PAGE];
+ let def_levels = vec![max_def_level; LARGE_VALUES_PER_PAGE];
+ // Every value is distinct, so the dictionary holds one entry per row and
each
+ // entry is referenced exactly once, as it is for a column of large unique
+ // payloads. The leading bytes make each value unique; the rest is filler.
+ let make_value = |index: usize| {
+ let mut value = vec![(index % 251) as u8; LARGE_VALUE_LEN];
+ value[..8].copy_from_slice(&(index as u64).to_le_bytes());
+ value
+ };
+ let mut next_value = 0;
+ let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
+ for _i in 0..NUM_ROW_GROUPS {
+ let mut column_chunk_pages = VecDeque::new();
+ let mut dict_encoder =
DictEncoder::<ByteArrayType>::new(column_desc.clone());
+ // add data pages
+ for _j in 0..PAGES_PER_GROUP {
+ let values = (0..LARGE_VALUES_PER_PAGE)
+ .map(|_| {
+ next_value += 1;
+ parquet::data_type::ByteArray::from(make_value(next_value
- 1))
+ })
+ .collect::<Vec<_>>();
+ let mut page_builder =
+ DataPageBuilderImpl::new(column_desc.clone(), values.len() as
u32, true);
+ page_builder.add_rep_levels(max_rep_level, &rep_levels);
+ page_builder.add_def_levels(max_def_level, &def_levels);
+ let _ = dict_encoder.put(&values);
+ let indices = dict_encoder
+ .write_indices()
+ .expect("write_indices() should be OK");
+ page_builder.add_indices(indices);
+ column_chunk_pages.push_back(page_builder.consume());
+ }
+ // add dictionary page
+ let dict = dict_encoder
+ .write_dict()
+ .expect("write_dict() should be OK");
+ let dict_page = parquet::column::page::Page::DictionaryPage {
+ buf: dict,
+ num_values: dict_encoder.num_entries() as u32,
+ encoding: Encoding::RLE_DICTIONARY,
+ is_sorted: false,
+ };
+ column_chunk_pages.push_front(dict_page);
+ pages.push(column_chunk_pages.into());
+ }
+
+ InMemoryPageIterator::new(pages)
+}
+
fn build_string_list_page_iterator(
column_desc: ColumnDescPtr,
null_density: f32,
@@ -2277,6 +2350,23 @@ fn add_benches(c: &mut Criterion) {
assert_eq!(count, EXPECTED_VALUE_COUNT);
});
+ // byte array, dictionary encoded, large values, no NULLs
+ let dictionary_large_value_data =
+
build_dictionary_encoded_large_value_page_iterator(mandatory_binary_column_desc.clone());
+ group.bench_function(
+ "dictionary encoded, mandatory, no NULLs, large values",
+ |b| {
+ b.iter(|| {
+ let array_reader = create_byte_array_reader(
+ dictionary_large_value_data.clone(),
+ mandatory_binary_column_desc.clone(),
+ );
+ count = bench_array_reader(array_reader);
+ });
+ assert_eq!(count, EXPECTED_LARGE_VALUE_COUNT);
+ },
+ );
+
group.finish();
// binary view benchmarks
