This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 5db31bc9bc perf: Reserve concat output space using visible offset span
(#11270)
5db31bc9bc is described below
commit 5db31bc9bce4acb246699477015cc83c27e0a0d0
Author: Neil Conway <[email protected]>
AuthorDate: Wed Sep 30 09:22:06 2026 -0400
perf: Reserve concat output space using visible offset span (#11270)
# Which issue does this PR close?
- N/A
# Rationale for this change
`concat_elements_bytes` and `concat_elements_utf8_many` sized their
output capacities using the size of the input's values array minus the
first offset. This can result in over-allocation when the input is
sliced to omit values at the suffix of the values array.
Benchmarks:
- Two inputs, compact: 48.79 → 48.94 µs (0.31% slower). Allocated output
value-buffer capacity is 469,824 bytes (unchanged).
- Three inputs, compact: 87.73 → 84.91 µs (3.21% less time). Output
capacity is 704,736 bytes (unchanged).
- Two inputs, sliced: 49.24 → 49.38 µs (0.27% slower). Capacity
7,078,784 → 469,824 bytes, a 93.36% reduction.
- Three inputs, sliced: 88.69 → 85.18 µs (3.96% less time). Capacity
10,618,176 → 704,736 bytes, a 93.36% reduction.
# What changes are included in this PR?
* Fix output sizing for `concat_elements_bytes` and
`concat_elements_utf8_many` for sliced inputs
* Add benchmark
* Add unit test
# Are these changes tested?
Yes; existing tests pass, new test added.
# Are there any user-facing changes?
No.
# AI usage
Developed with Codex (Astra 6), reviewed and revised with Claude Code
(Opus 5.5). I have reviewed and understand the resulting code.
---
arrow-string/src/concat_elements.rs | 28 ++++++++++++++++++++--------
arrow/benches/concatenate_elements.rs | 17 ++++++++++++++++-
2 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/arrow-string/src/concat_elements.rs
b/arrow-string/src/concat_elements.rs
index 0df4216c66..0a7e76f045 100644
--- a/arrow-string/src/concat_elements.rs
+++ b/arrow-string/src/concat_elements.rs
@@ -47,16 +47,15 @@ pub fn concat_elements_bytes<T: ByteArrayType>(
let nulls = NullBuffer::union(left.nulls(), right.nulls());
- let left_offsets = left.value_offsets();
- let right_offsets = right.value_offsets();
+ let left_offsets = left.offsets();
+ let right_offsets = right.offsets();
let left_values = left.value_data();
let right_values = right.value_data();
let mut output_values = Vec::with_capacity(
- left_values.len() + right_values.len()
- - left_offsets[0].as_usize()
- - right_offsets[0].as_usize(),
+ (left_offsets.last() - left_offsets.first()).as_usize()
+ + (right_offsets.last() - right_offsets.first()).as_usize(),
);
let mut output_offsets = Vec::with_capacity(left_offsets.len());
@@ -154,10 +153,9 @@ pub fn concat_elements_utf8_many<Offset: OffsetSizeTrait>(
.collect::<Vec<_>>();
let mut output_values = Vec::with_capacity(
- data_values
+ arrays
.iter()
- .zip(offsets.iter_mut())
- .map(|(data, offset)| data.len() -
offset.peek().unwrap().as_usize())
+ .map(|array| (array.offsets().last() -
array.offsets().first()).as_usize())
.sum(),
);
@@ -608,6 +606,20 @@ mod tests {
assert_eq!(output.as_string(), &expected);
}
+ #[test]
+ fn test_concat_slice_capacity() {
+ let left = StringArray::from(vec!["hello"; 8]).slice(2, 3);
+ let right = StringArray::from(vec![" world"; 8]).slice(4, 3);
+
+ let pair = concat_elements_utf8(&left, &right).unwrap();
+ assert_eq!(pair, StringArray::from(vec!["hello world"; 3]));
+ assert_eq!(pair.values().capacity(), 33);
+
+ let many = concat_elements_utf8_many(&[&left, &right, &left]).unwrap();
+ assert_eq!(many, StringArray::from(vec!["hello worldhello"; 3]));
+ assert_eq!(many.values().capacity(), 48);
+ }
+
#[test]
fn test_string_concat_error_empty() {
assert_eq!(
diff --git a/arrow/benches/concatenate_elements.rs
b/arrow/benches/concatenate_elements.rs
index fcacbe3284..33197cd11e 100644
--- a/arrow/benches/concatenate_elements.rs
+++ b/arrow/benches/concatenate_elements.rs
@@ -22,7 +22,7 @@ use criterion::Criterion;
use arrow::array::*;
use arrow::util::bench_util::*;
-use arrow_string::concat_elements::concat_elements_dyn;
+use arrow_string::concat_elements::{concat_elements_dyn,
concat_elements_utf8_many};
use std::hint;
fn bench_concat(v1: &dyn Array, v2: &dyn Array) {
@@ -68,6 +68,21 @@ fn add_benchmark(c: &mut Criterion) {
c.bench_function(&id, |b| b.iter(|| bench_concat(&array, &array)));
}
}
+
+ // A slice whose values buffer extends far past its last row, compared with
+ // a compact copy of the same rows
+ let sliced = create_string_array_with_len::<i32>(131_072, 0.1,
32).slice(8_192, 8_192);
+ let compact = StringArray::from_iter(sliced.iter());
+ for (name, array) in [("compact", compact), ("sliced", sliced)] {
+ c.bench_function(&format!("concat str {name} 8192"), |b| {
+ b.iter(|| bench_concat(&array, &array))
+ });
+ c.bench_function(&format!("concat str many {name} 8192"), |b| {
+ b.iter(|| {
+ hint::black_box(concat_elements_utf8_many(&[&array, &array,
&array]).unwrap())
+ })
+ });
+ }
}
criterion_group!(benches, add_benchmark);
