comphead commented on code in PR #12058:
URL: https://github.com/apache/arrow-rs/pull/12058#discussion_r4634462792
##########
arrow/benches/arithmetic_kernels.rs:
##########
@@ -60,5 +66,105 @@ fn add_benchmark(c: &mut Criterion) {
}
}
-criterion_group!(benches, add_benchmark);
+/// Like `create_primitive_array_range`, but with its own `seed`, so that two
+/// arrays hold different values and nulls.
+fn seeded_array<T>(
Review Comment:
`seeded_array` is `create_primitive_array_range` with a seed, and #12062
adds the same copy to this file. Could it move to
`arrow/src/util/bench_util.rs` as `create_primitive_array_range_with_seed`,
next to `create_primitive_array_with_seed`? Both PRs could then use one copy,
and this file would stop conflicting between them.
##########
arrow/benches/arithmetic_kernels.rs:
##########
@@ -60,5 +66,105 @@ fn add_benchmark(c: &mut Criterion) {
}
}
-criterion_group!(benches, add_benchmark);
+/// Like `create_primitive_array_range`, but with its own `seed`, so that two
+/// arrays hold different values and nulls.
+fn seeded_array<T>(
+ size: usize,
+ null_density: f32,
+ range: Range<T::Native>,
+ seed: u64,
+) -> PrimitiveArray<T>
+where
+ T: ArrowPrimitiveType,
+ T::Native: SampleUniform,
+{
+ let mut rng = StdRng::seed_from_u64(seed);
+ (0..size)
+ .map(|_| (rng.random::<f32>() >= null_density).then(||
rng.random_range(range.clone())))
+ .collect()
+}
+
+/// Checked and wrapping integer arithmetic at DataFusion's default batch size.
+///
+/// The left operands come from `a_range` and the right ones from `b_range`, so
+/// the checked cases never overflow in a valid row unless the case name says
+/// so. Null slots hold 0, as a builder writes them, so with unsigned types
+/// `subtract_checked_scalar` overflows in null slots.
+fn integer_benchmark<T>(
+ c: &mut Criterion,
+ name: &str,
+ a_range: Range<T::Native>,
+ b_range: Range<T::Native>,
+ max: T::Native,
Review Comment:
nit: every caller passes `T::Native::MAX_TOTAL_ORDER` for `max`, which the
overflow-position test already uses, so the parameter could go. On line 146,
`PrimitiveArray::<T>::new_scalar(one)` is shorter than
`Scalar::new(PrimitiveArray::<T>::from_iter_values([one]))`.
##########
arrow-arith/src/numeric.rs:
##########
@@ -1693,6 +1913,281 @@ mod tests {
assert_eq!(err, "Divide by zero error");
}
+ type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+ type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+ /// Checks `add`, `sub` and `mul` against the scalar checked operations for
+ /// every pair of `edges`, with arrays on both sides and with a scalar on
+ /// either side.
+ fn test_checked_integer_ops<T: ArrowPrimitiveType>(edges: &[T::Native]) {
+ test_checked_integer_op::<T>(edges, add, |l, r| l.add_checked(r));
+ test_checked_integer_op::<T>(edges, sub, |l, r| l.sub_checked(r));
+ test_checked_integer_op::<T>(edges, mul, |l, r| l.mul_checked(r));
+ }
+
+ fn test_checked_integer_op<T: ArrowPrimitiveType>(
+ edges: &[T::Native],
+ kernel: Kernel,
+ checked: Checked<T::Native>,
+ ) {
+ let (l, r): (Vec<_>, Vec<_>) = edges
+ .iter()
+ .flat_map(|&x| edges.iter().map(move |&y| (x, y)))
+ .unzip();
+ // With every third row null, null slots also hold values that
overflow.
+ // Without nulls, every pair is checked, because when the number of
+ // edges is a multiple of 3 the null rows always pair with the same
+ // right-hand values.
+ let every_third = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 !=
2));
+ for nulls in [Some(every_third), None] {
+ let valid = |i| nulls.as_ref().is_none_or(|n| n.is_valid(i));
+ let l_array = PrimitiveArray::<T>::new(l.clone().into(),
nulls.clone());
+ let r_array =
PrimitiveArray::<T>::from_iter_values(r.iter().copied());
+ let context = if nulls.is_some() { "nulls" } else { "no nulls" };
+
+ // The kernels report the first valid row that overflows.
+ let expected = (0..l.len())
+ .map(|i| valid(i).then(|| checked(l[i], r[i])).transpose())
+ .collect();
+ assert_kernel_result::<T>(
+ kernel(&l_array, &r_array),
+ expected,
+ &format!("{context}: array, array"),
+ );
+
+ // An overflow in one row sends the whole array to the checked
kernel,
+ // which would hide a missed overflow in another row, so also
check each
+ // row alone.
+ for i in 0..l.len() {
+ let expected = match valid(i) {
+ true => checked(l[i], r[i]).map(|v| vec![Some(v)]),
+ false => Ok(vec![None]),
+ };
+ let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i,
1));
+ assert_kernel_result::<T>(actual, expected,
&format!("{context}: row {i}"));
+ }
+
+ for &x in edges {
+ let scalar = PrimitiveArray::<T>::new_scalar(x);
+ let expected = r.iter().map(|&y| checked(x,
y).map(Some)).collect();
+ assert_kernel_result::<T>(
+ kernel(&scalar, &r_array),
+ expected,
+ &format!("{context}: scalar {x:?}, array"),
+ );
+ let expected = (0..l.len())
+ .map(|i| valid(i).then(|| checked(l[i], x)).transpose())
+ .collect();
+ assert_kernel_result::<T>(
+ kernel(&l_array, &scalar),
+ expected,
+ &format!("{context}: array, scalar {x:?}"),
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn test_checked_integer_edges() {
+ macro_rules! signed {
+ ($a:ty, $t:ty) => {
+ test_checked_integer_ops::<$a>(&[
+ <$t>::MIN,
+ <$t>::MIN + 1,
+ <$t>::MIN / 2,
+ -3,
+ -1,
+ 0,
+ 1,
+ 3,
+ <$t>::MAX / 2,
+ <$t>::MAX / 2 + 1,
+ <$t>::MAX - 1,
+ <$t>::MAX,
+ ])
+ };
+ }
+ macro_rules! unsigned {
+ ($a:ty, $t:ty) => {
+ test_checked_integer_ops::<$a>(&[
+ 0,
+ 1,
+ 3,
+ <$t>::MAX / 2,
+ <$t>::MAX / 2 + 1,
+ <$t>::MAX - 1,
+ <$t>::MAX,
+ ])
+ };
+ }
+ signed!(Int8Type, i8);
+ signed!(Int16Type, i16);
+ signed!(Int32Type, i32);
+ signed!(Int64Type, i64);
+ unsigned!(UInt8Type, u8);
+ unsigned!(UInt16Type, u16);
+ unsigned!(UInt32Type, u32);
+ unsigned!(UInt64Type, u64);
+ }
+
+ /// Moves one row that overflows through a batch, so that each lane of the
+ /// vector loop and each row of the scalar remainder holds it once, with an
+ /// array and a scalar on either side. The same row is also checked as a
+ /// null slot, which is not an error.
+ fn test_checked_integer_overflow_position<T: ArrowPrimitiveType>() {
+ // Two iterations of the widest vector loop we build, 32 `i8` lanes of
+ // AVX2 unrolled four times, plus a remainder.
+ const LEN: usize = 300;
+ let one = T::Native::ONE;
+ // The kernel, its checked scalar operation, the value that overflows,
+ // and the other operand.
+ type Case<N> = (Kernel, Checked<N>, N, N);
+ let cases: [Case<T::Native>; 3] = [
+ (
+ add,
+ |l, r| l.add_checked(r),
+ T::Native::MAX_TOTAL_ORDER,
+ one,
+ ),
+ (
+ sub,
+ |l, r| l.sub_checked(r),
+ T::Native::MIN_TOTAL_ORDER,
+ one,
+ ),
+ (
+ mul,
+ |l, r| l.mul_checked(r),
+ T::Native::MAX_TOTAL_ORDER,
+ one.add_wrapping(one),
+ ),
+ ];
+ for (kernel, checked, edge, other) in cases {
+ let other_array = PrimitiveArray::<T>::from_value(other, LEN);
+ let other_scalar = PrimitiveArray::<T>::new_scalar(other);
+ for i in 0..LEN {
+ let mut values = vec![one; LEN];
Review Comment:
nit: `values` is allocated again for every `i`. It could be allocated once
before the loop, with `values[i] = edge` before the inner loop and `values[i] =
one` after it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]