Rich-T-kid commented on code in PR #10813:
URL: https://github.com/apache/arrow-rs/pull/10813#discussion_r3865995502
##########
arrow-select/src/take.rs:
##########
@@ -462,37 +462,169 @@ fn take_bits<I: ArrowPrimitiveType>(
indices: &PrimitiveArray<I>,
) -> BooleanBuffer {
let len = indices.len();
+ let src_offset = values.offset();
+ let src_ptr = values.values().as_ptr();
+ let out_bytes = len.div_ceil(8);
match indices.nulls().filter(|n| n.null_count() > 0) {
Some(nulls) => {
- let mut output_buffer = MutableBuffer::new_null(len);
- let output_slice = output_buffer.as_slice_mut();
- nulls.valid_indices().for_each(|idx| {
- // SAFETY: idx is a valid index in indices.nulls() -->
idx<indices.len()
- if values.value(unsafe {
indices.value_unchecked(idx).as_usize() }) {
- // SAFETY: MutableBuffer was created with space for
indices.len() bit, and idx < indices.len()
- unsafe { bit_util::set_bit_raw(output_slice.as_mut_ptr(),
idx) };
+ let mut output = MutableBuffer::new_null(len);
+ let out_ptr = output.as_mut_ptr();
+ nulls.valid_indices().for_each(|i| {
+ // SAFETY: i < len from the validity bitmap
+ let src_idx = unsafe { indices.value_unchecked(i) }.as_usize()
+ src_offset;
+ // SAFETY: src_idx bounded by take's prior bounds check
+ unsafe {
+ if bit_util::get_bit_raw(src_ptr, src_idx) {
+ bit_util::set_bit_raw(out_ptr, i);
+ }
}
});
- BooleanBuffer::new(output_buffer.into(), 0, len)
+ BooleanBuffer::new(output.into(), 0, len)
}
None => {
- BooleanBuffer::collect_bool(len, |idx: usize| {
- // SAFETY: idx<indices.len()
- values.value(unsafe { indices.value_unchecked(idx).as_usize()
})
- })
+ // Build the output byte-by-byte with an 8-element inner loop so
the
+ // compiler can fully unroll it and issue the 8 source loads in
parallel.
+ let mut output = MutableBuffer::with_capacity(out_bytes);
+ // SAFETY: every byte is written before BooleanBuffer reads it
+ unsafe { output.set_len(out_bytes) };
+ let out_slice = output.as_slice_mut();
+ let full_bytes = len / 8;
+
+ for byte_idx in 0..full_bytes {
+ let base = byte_idx * 8;
+ let mut byte = 0u8;
+ for bit in 0..8usize {
+ // SAFETY: base + bit < len
+ let src_idx =
+ unsafe { indices.value_unchecked(base + bit)
}.as_usize() + src_offset;
+ // SAFETY: src_idx bounded by take's prior bounds check
+ let raw = unsafe { *src_ptr.add(src_idx >> 3) };
+ byte |= ((raw >> (src_idx & 7)) & 1) << bit;
+ }
+ out_slice[byte_idx] = byte;
+ }
+ if full_bytes < out_bytes {
+ let base = full_bytes * 8;
+ let mut byte = 0u8;
+ for bit in 0..(len - base) {
+ let src_idx =
+ unsafe { indices.value_unchecked(base + bit)
}.as_usize() + src_offset;
+ let raw = unsafe { *src_ptr.add(src_idx >> 3) };
+ byte |= ((raw >> (src_idx & 7)) & 1) << bit;
+ }
+ out_slice[full_bytes] = byte;
+ }
+ BooleanBuffer::new(output.into(), 0, len)
}
}
}
-/// `take` implementation for boolean arrays
+/// Gather value bits and validity bits from two boolean buffers in a single
pass.
+/// Used when the values array itself has nulls, avoiding two separate
`take_bits` calls.
+#[inline(never)]
+fn take_bits_with_validity<I: ArrowPrimitiveType>(
+ values: &BooleanBuffer,
+ validity: &BooleanBuffer,
+ indices: &PrimitiveArray<I>,
Review Comment:
we may be able to avoid extra allocations we we use
https://arrow.apache.org/rust/arrow_buffer/buffer/immutable/struct.Buffer.html#method.into_mutable
##########
arrow-select/src/take.rs:
##########
@@ -462,37 +462,169 @@ fn take_bits<I: ArrowPrimitiveType>(
indices: &PrimitiveArray<I>,
) -> BooleanBuffer {
let len = indices.len();
+ let src_offset = values.offset();
+ let src_ptr = values.values().as_ptr();
+ let out_bytes = len.div_ceil(8);
match indices.nulls().filter(|n| n.null_count() > 0) {
Some(nulls) => {
- let mut output_buffer = MutableBuffer::new_null(len);
- let output_slice = output_buffer.as_slice_mut();
- nulls.valid_indices().for_each(|idx| {
- // SAFETY: idx is a valid index in indices.nulls() -->
idx<indices.len()
- if values.value(unsafe {
indices.value_unchecked(idx).as_usize() }) {
- // SAFETY: MutableBuffer was created with space for
indices.len() bit, and idx < indices.len()
- unsafe { bit_util::set_bit_raw(output_slice.as_mut_ptr(),
idx) };
+ let mut output = MutableBuffer::new_null(len);
+ let out_ptr = output.as_mut_ptr();
+ nulls.valid_indices().for_each(|i| {
+ // SAFETY: i < len from the validity bitmap
+ let src_idx = unsafe { indices.value_unchecked(i) }.as_usize()
+ src_offset;
+ // SAFETY: src_idx bounded by take's prior bounds check
+ unsafe {
+ if bit_util::get_bit_raw(src_ptr, src_idx) {
+ bit_util::set_bit_raw(out_ptr, i);
+ }
}
});
- BooleanBuffer::new(output_buffer.into(), 0, len)
+ BooleanBuffer::new(output.into(), 0, len)
}
None => {
- BooleanBuffer::collect_bool(len, |idx: usize| {
- // SAFETY: idx<indices.len()
- values.value(unsafe { indices.value_unchecked(idx).as_usize()
})
- })
+ // Build the output byte-by-byte with an 8-element inner loop so
the
+ // compiler can fully unroll it and issue the 8 source loads in
parallel.
+ let mut output = MutableBuffer::with_capacity(out_bytes);
+ // SAFETY: every byte is written before BooleanBuffer reads it
+ unsafe { output.set_len(out_bytes) };
+ let out_slice = output.as_slice_mut();
+ let full_bytes = len / 8;
+
+ for byte_idx in 0..full_bytes {
+ let base = byte_idx * 8;
+ let mut byte = 0u8;
+ for bit in 0..8usize {
+ // SAFETY: base + bit < len
+ let src_idx =
+ unsafe { indices.value_unchecked(base + bit)
}.as_usize() + src_offset;
+ // SAFETY: src_idx bounded by take's prior bounds check
+ let raw = unsafe { *src_ptr.add(src_idx >> 3) };
+ byte |= ((raw >> (src_idx & 7)) & 1) << bit;
+ }
+ out_slice[byte_idx] = byte;
+ }
+ if full_bytes < out_bytes {
+ let base = full_bytes * 8;
+ let mut byte = 0u8;
+ for bit in 0..(len - base) {
+ let src_idx =
+ unsafe { indices.value_unchecked(base + bit)
}.as_usize() + src_offset;
+ let raw = unsafe { *src_ptr.add(src_idx >> 3) };
+ byte |= ((raw >> (src_idx & 7)) & 1) << bit;
+ }
+ out_slice[full_bytes] = byte;
+ }
+ BooleanBuffer::new(output.into(), 0, len)
}
}
}
-/// `take` implementation for boolean arrays
+/// Gather value bits and validity bits from two boolean buffers in a single
pass.
+/// Used when the values array itself has nulls, avoiding two separate
`take_bits` calls.
+#[inline(never)]
+fn take_bits_with_validity<I: ArrowPrimitiveType>(
+ values: &BooleanBuffer,
+ validity: &BooleanBuffer,
+ indices: &PrimitiveArray<I>,
Review Comment:
swapping from mutableBuffer to `Vec<u8>`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
