viirya commented on a change in pull request #1248:
URL: https://github.com/apache/arrow-rs/pull/1248#discussion_r801063167
##########
File path: arrow/src/compute/kernels/filter.rs
##########
@@ -17,24 +17,60 @@
//! Defines miscellaneous array kernels.
+use std::ops::AddAssign;
+use std::sync::Arc;
+
+use num::Zero;
+
+use TimeUnit::*;
+
use crate::array::*;
-use crate::buffer::buffer_bin_and;
-use crate::datatypes::DataType;
-use crate::error::Result;
+use crate::buffer::{buffer_bin_and, Buffer, MutableBuffer};
+use crate::datatypes::*;
+use crate::error::{ArrowError, Result};
use crate::record_batch::RecordBatch;
use crate::util::bit_chunk_iterator::{UnalignedBitChunk,
UnalignedBitChunkIterator};
+use crate::util::bit_util;
-/// Function that can filter arbitrary arrays
-pub type Filter<'a> = Box<dyn Fn(&ArrayData) -> ArrayData + 'a>;
+/// If the filter selects more than this fraction of rows, use
+/// [`SlicesIterator`] to copy ranges of values. Otherwise iterate
+/// over individual rows using [`IndexIterator`]
+///
+/// Threshold of 0.8 chosen based on
<https://dl.acm.org/doi/abs/10.1145/3465998.3466009>
+///
+const FILTER_SLICES_SELECTIVITY_THRESHOLD: f64 = 0.8;
+
+macro_rules! downcast_filter {
+ ($type: ty, $values: expr, $filter: expr) => {{
+ let values = $values
+ .as_any()
+ .downcast_ref::<PrimitiveArray<$type>>()
+ .expect("Unable to downcast to a primitive array");
+
+ Ok(Arc::new(filter_primitive::<$type>(&values, $filter)))
+ }};
+}
+
+macro_rules! downcast_dict_filter {
+ ($type: ty, $values: expr, $filter: expr) => {{
+ let values = $values
+ .as_any()
+ .downcast_ref::<DictionaryArray<$type>>()
+ .expect("Unable to downcast to a dictionary array");
+ Ok(Arc::new(filter_dict::<$type>(values, $filter)))
+ }};
+}
-/// An iterator of `(usize, usize)` each representing an interval
`[start,end[` whose
-/// slots of a [BooleanArray] are true. Each interval corresponds to a
contiguous region of memory to be
-/// "taken" from an array to be filtered.
+/// An iterator of `(usize, usize)` each representing an interval `[start,
end]` whose
+/// slots of a [BooleanArray] are true. Each interval corresponds to a
contiguous region of memory
+/// to be "taken" from an array to be filtered.
+///
+/// This is only performant for the least selective filters that copy across
long contiguous runs
Review comment:
ditto, I think this should be highly selective filters?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
