This is an automated email from the ASF dual-hosted git repository.
Jefffrey pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 9de5b5f34c Replace concat_elements_utf8_many BufferBuilders with Vec
(#10633)
9de5b5f34c is described below
commit 9de5b5f34c194bc4bbd5139ea8fbb858e8525f1c
Author: cakeni <[email protected]>
AuthorDate: Thu Aug 20 09:24:33 2026 +0800
Replace concat_elements_utf8_many BufferBuilders with Vec (#10633)
# Which issue does this PR close?
- Part of #10245.
# Rationale for this change
Using `Vec` instead of `BufferBuilder` can benefit from Rust's optimized
vector implementation. This updates the value and offset builders in
`concat_elements_utf8_many`.
# What changes are included in this PR?
- Replace the many-array output value and offset builders with
capacity-matched vectors.
- Use `extend_from_slice` and `push` while preserving the existing
offset iteration.
- Convert the vectors directly into the buffers used by
`ArrayDataBuilder`.
- Leave the separate two-array concatenation path unchanged.
# Are these changes tested?
Yes. The following checks pass:
- `cargo +stable-x86_64-pc-windows-gnu fmt --all -- --check`
- `cargo +stable-x86_64-pc-windows-gnu clippy -p arrow-string
--all-targets --all-features --no-deps -- -D warnings`
- `cargo +stable-x86_64-pc-windows-gnu test -p arrow-string
--all-features` (182 unit tests and 10 doctests passed)
# Are there any user-facing changes?
No.
## AI assistance
OpenAI Codex assisted with exploring the relevant code and drafting this
optimization. I reviewed the final changes.
Co-authored-by: Jeffrey Vo <[email protected]>
---
arrow-string/src/concat_elements.rs | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/arrow-string/src/concat_elements.rs
b/arrow-string/src/concat_elements.rs
index 62c0c9ffce..0178be8e80 100644
--- a/arrow-string/src/concat_elements.rs
+++ b/arrow-string/src/concat_elements.rs
@@ -140,7 +140,7 @@ pub fn concat_elements_utf8_many<Offset: OffsetSizeTrait>(
.map(|a| a.value_offsets().iter().peekable())
.collect::<Vec<_>>();
- let mut output_values = BufferBuilder::<u8>::new(
+ let mut output_values = Vec::with_capacity(
data_values
.iter()
.zip(offsets.iter_mut())
@@ -148,8 +148,8 @@ pub fn concat_elements_utf8_many<Offset: OffsetSizeTrait>(
.sum(),
);
- let mut output_offsets = BufferBuilder::<Offset>::new(size + 1);
- output_offsets.append(Offset::zero());
+ let mut output_offsets = Vec::with_capacity(size + 1);
+ output_offsets.push(Offset::zero());
for _ in 0..size {
data_values
.iter()
@@ -157,15 +157,15 @@ pub fn concat_elements_utf8_many<Offset: OffsetSizeTrait>(
.for_each(|(values, offset)| {
let index_start = offset.next().unwrap().as_usize();
let index_end = offset.peek().unwrap().as_usize();
- output_values.append_slice(&values[index_start..index_end]);
+
output_values.extend_from_slice(&values[index_start..index_end]);
});
-
output_offsets.append(Offset::from_usize(output_values.len()).unwrap());
+ output_offsets.push(Offset::from_usize(output_values.len()).unwrap());
}
let builder =
ArrayDataBuilder::new(GenericStringArray::<Offset>::DATA_TYPE)
.len(size)
- .add_buffer(output_offsets.finish())
- .add_buffer(output_values.finish())
+ .add_buffer(output_offsets.into())
+ .add_buffer(output_values.into())
.nulls(nulls);
// SAFETY - offsets valid by construction
