Rich-T-kid commented on code in PR #11871:
URL: https://github.com/apache/arrow-rs/pull/11871#discussion_r4367747589
##########
arrow/benches/filter_kernels.rs:
##########
@@ -37,6 +41,54 @@ fn bench_built_filter(array: &dyn Array, filter:
&FilterPredicate) {
hint::black_box(filter.filter(array).unwrap());
}
+/// Creates a `Struct<x: Int64, y: Utf8>` array of `len` rows
+fn create_xy_struct_array(len: usize) -> StructArray {
+ let fields = Fields::from(vec![
+ Field::new("x", DataType::Int64, true),
+ Field::new("y", DataType::Utf8, true),
+ ]);
+ let x = create_primitive_array::<Int64Type>(len, 0.1);
+ let y = create_string_array_with_max_len::<i32>(len, 0.1, 16);
+ StructArray::new(fields, vec![Arc::new(x), Arc::new(y)], None)
+}
+
+/// Creates a `List<Struct<x: Int64, y: Utf8>>` array of `size` rows, each
+/// with a random number of elements in `0..=max_list_size`
+fn create_struct_list_array(size: usize, null_density: f32, max_list_size:
usize) -> ListArray {
+ let mut rng = StdRng::seed_from_u64(42);
+ let lengths: Vec<usize> = (0..size)
+ .map(|_| rng.random_range(0..=max_list_size))
+ .collect();
+ let values = create_xy_struct_array(lengths.iter().sum());
+ let offsets = OffsetBuffer::<i32>::from_lengths(lengths);
+ let nulls = (null_density > 0.0)
+ .then(|| NullBuffer::from_iter((0..size).map(|_| rng.random::<f32>()
>= null_density)));
+ let field = Arc::new(Field::new_list_field(values.data_type().clone(),
true));
+ ListArray::new(field, offsets, Arc::new(values), nulls)
+}
+
+/// Creates a `FixedSizeList<Struct<x: Int64, y: Utf8>>` array of `size` rows
+fn create_struct_fixed_size_list_array(size: usize, list_size: i32) ->
FixedSizeListArray {
+ let values = create_xy_struct_array(size * list_size as usize);
+ let field = Arc::new(Field::new_list_field(values.data_type().clone(),
true));
+ FixedSizeListArray::new(field, list_size, Arc::new(values), None)
+}
+
+/// Creates the `Struct<a..h: Int64, items: List<Struct<x: Int64, y: Utf8>>>`
+/// array from <https://github.com/apache/datafusion/issues/26581>
+fn create_struct_with_list_array(size: usize, max_list_size: usize) ->
StructArray {
+ let mut fields = vec![];
+ let mut columns: Vec<ArrayRef> = vec![];
+ for name in ["a", "b", "c", "d", "e", "f", "g", "h"] {
+ fields.push(Field::new(name, DataType::Int64, true));
+ columns.push(Arc::new(create_primitive_array::<Int64Type>(size, 0.1)));
+ }
+ let items = create_struct_list_array(size, 0.1, max_list_size);
+ fields.push(Field::new("items", items.data_type().clone(), true));
+ columns.push(Arc::new(items));
+ StructArray::new(Fields::from(fields), columns, None)
+}
Review Comment:
The rest of the benchmarks seem pretty straightforward, but this one is a
bit more complex. Do we have a design in mind that would make filtering nested
data like this more efficient?
I’m asking because if the implementation follows [other patterns like the
take
kernel](https://github.com/apache/arrow-rs/blob/34bf27befe8ed252f5ac826799cda05c15c1dd6d/arrow-select/src/take.rs#L306),
where struct fields just call take on each individual array/column, then we
won’t actually be benchmarking anything unique to this array type.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
