Dandandan commented on code in PR #7873:
URL: https://github.com/apache/arrow-rs/pull/7873#discussion_r2195253620
##########
arrow-array/src/array/byte_view_array.rs:
##########
@@ -473,13 +473,92 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// Note: this function does not attempt to canonicalize / deduplicate
values. For this
/// feature see [`GenericByteViewBuilder::with_deduplicate_strings`].
pub fn gc(&self) -> Self {
- let mut builder =
GenericByteViewBuilder::<T>::with_capacity(self.len());
+ // 1) Read basic properties once
+ let len = self.len(); // number of elements
+ let views = self.views(); // raw u128 "view" values per slot
+ let nulls = self.nulls().cloned(); // reuse & clone existing null
bitmap
+
+ // 2) Calculate total size of all non-inline data and detect if any
exists
+ let mut total_large = 0;
+ if let Some(nbm) = &nulls {
+ for i in nbm.valid_indices() {
+ let bv = ByteView::from(unsafe { *views.get_unchecked(i) });
+ if bv.length > MAX_INLINE_VIEW_LEN {
+ total_large += bv.length as usize;
+ }
+ }
+ } else {
+ for &raw in views.iter() {
+ let bv = ByteView::from(raw);
+ if bv.length > MAX_INLINE_VIEW_LEN {
+ total_large += bv.length as usize;
+ }
+ }
+ }
- for v in self.iter() {
- builder.append_option(v);
+ // 2.5) Fast path: if there is no non-inline data, avoid buffer
allocation & processing
+ if total_large == 0 {
+ // Views are inline-only or all null; just reuse original views
and no data blocks
+ let views_scalar = ScalarBuffer::from(views.to_vec());
Review Comment:
I think `to_vec` allocates - I think we could just `clone` the `views` to
create a new one?
##########
arrow-array/src/array/byte_view_array.rs:
##########
@@ -473,13 +473,92 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// Note: this function does not attempt to canonicalize / deduplicate
values. For this
/// feature see [`GenericByteViewBuilder::with_deduplicate_strings`].
pub fn gc(&self) -> Self {
- let mut builder =
GenericByteViewBuilder::<T>::with_capacity(self.len());
+ // 1) Read basic properties once
+ let len = self.len(); // number of elements
+ let views = self.views(); // raw u128 "view" values per slot
+ let nulls = self.nulls().cloned(); // reuse & clone existing null
bitmap
+
+ // 2) Calculate total size of all non-inline data and detect if any
exists
+ let mut total_large = 0;
+ if let Some(nbm) = &nulls {
+ for i in nbm.valid_indices() {
+ let bv = ByteView::from(unsafe { *views.get_unchecked(i) });
+ if bv.length > MAX_INLINE_VIEW_LEN {
+ total_large += bv.length as usize;
+ }
+ }
+ } else {
+ for &raw in views.iter() {
+ let bv = ByteView::from(raw);
+ if bv.length > MAX_INLINE_VIEW_LEN {
+ total_large += bv.length as usize;
+ }
+ }
+ }
- for v in self.iter() {
- builder.append_option(v);
+ // 2.5) Fast path: if there is no non-inline data, avoid buffer
allocation & processing
+ if total_large == 0 {
+ // Views are inline-only or all null; just reuse original views
and no data blocks
+ let views_scalar = ScalarBuffer::from(views.to_vec());
Review Comment:
I think `to_vec` allocates to a new `Vec` - I think we could just `clone`
the `views` to create a new one?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
