mbutrovich commented on code in PR #12058:
URL: https://github.com/apache/arrow-rs/pull/12058#discussion_r4568382506
##########
arrow-arith/src/numeric.rs:
##########
@@ -1693,6 +1839,165 @@ mod tests {
assert_eq!(err, "Divide by zero error");
}
+ type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+ type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+ /// Checks `add`, `sub` and `mul` against the scalar checked operations for
+ /// every pair of `edges`, with arrays on both sides and with a scalar on
+ /// either side. Every third row of the left array is null, so null slots
+ /// also hold values that overflow.
+ fn test_checked_integer_ops<T: ArrowPrimitiveType>(edges: &[T::Native]) {
+ test_checked_integer_op::<T>(edges, add, |l, r| l.add_checked(r));
+ test_checked_integer_op::<T>(edges, sub, |l, r| l.sub_checked(r));
+ test_checked_integer_op::<T>(edges, mul, |l, r| l.mul_checked(r));
+ }
+
+ fn test_checked_integer_op<T: ArrowPrimitiveType>(
+ edges: &[T::Native],
+ kernel: Kernel,
+ checked: Checked<T::Native>,
+ ) {
+ let (l, r): (Vec<_>, Vec<_>) = edges
+ .iter()
+ .flat_map(|&x| edges.iter().map(move |&y| (x, y)))
+ .unzip();
+ let nulls = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 != 2));
+ let l_array = PrimitiveArray::<T>::new(l.clone().into(),
Some(nulls.clone()));
+ let r_array = PrimitiveArray::<T>::from_iter_values(r.iter().copied());
+
+ // The kernels report the first valid row that overflows.
+ let expected = (0..l.len())
+ .map(|i| nulls.is_valid(i).then(|| checked(l[i],
r[i])).transpose())
+ .collect();
+ assert_kernel_result::<T>(kernel(&l_array, &r_array), expected,
"array, array");
+
+ // An overflow in one row sends the whole array to the checked kernel,
+ // which would hide a missed overflow in another row, so also check
each
+ // row alone.
+ for i in 0..l.len() {
+ let expected = match nulls.is_valid(i) {
+ true => checked(l[i], r[i]).map(|v| vec![Some(v)]),
+ false => Ok(vec![None]),
+ };
+ let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i, 1));
+ assert_kernel_result::<T>(actual, expected, &format!("row {i}"));
+ }
+
+ for &x in edges {
+ let scalar = PrimitiveArray::<T>::new_scalar(x);
+ let expected = r.iter().map(|&y| checked(x,
y).map(Some)).collect();
+ assert_kernel_result::<T>(
+ kernel(&scalar, &r_array),
+ expected,
+ &format!("scalar {x:?}, array"),
+ );
+ let expected = (0..l.len())
+ .map(|i| nulls.is_valid(i).then(|| checked(l[i],
x)).transpose())
+ .collect();
+ assert_kernel_result::<T>(
+ kernel(&l_array, &scalar),
+ expected,
+ &format!("array, scalar {x:?}"),
+ );
+ }
+ }
+
+ #[test]
+ fn test_checked_integer_edges() {
+ macro_rules! signed {
+ ($a:ty, $t:ty) => {
+ test_checked_integer_ops::<$a>(&[
+ <$t>::MIN,
+ <$t>::MIN + 1,
+ <$t>::MIN / 2,
+ -3,
+ -1,
+ 0,
+ 1,
+ 3,
+ <$t>::MAX / 2,
+ <$t>::MAX / 2 + 1,
+ <$t>::MAX - 1,
+ <$t>::MAX,
+ ])
+ };
+ }
+ macro_rules! unsigned {
+ ($a:ty, $t:ty) => {
+ test_checked_integer_ops::<$a>(&[
+ 0,
+ 1,
+ 3,
+ <$t>::MAX / 2,
+ <$t>::MAX / 2 + 1,
+ <$t>::MAX - 1,
+ <$t>::MAX,
+ ])
+ };
+ }
+ signed!(Int8Type, i8);
+ signed!(Int16Type, i16);
+ signed!(Int32Type, i32);
+ signed!(Int64Type, i64);
+ unsigned!(UInt8Type, u8);
+ unsigned!(UInt16Type, u16);
+ unsigned!(UInt32Type, u32);
+ unsigned!(UInt64Type, u64);
+ }
+
+ #[test]
+ fn test_checked_integer_overflow_position() {
+ let one = Int32Array::new_scalar(1);
+ // Every length up to 130 puts the overflow in each part of the vector
loop
+ // for strides up to 64 rows.
+ for len in 1..=130 {
+ let mut values = vec![1; len];
+ values[len - 1] = i32::MAX;
Review Comment:
> Could the overflow position vary independently of the batch length?
Yes. `test_checked_integer_overflow_positions` now moves one overflowing row
through a 300-row batch for every integer type. It covers `add`, `sub`, and
`mul` with the batch against an array and against a scalar, on either side,
with that row valid and with it null. 300 rows cover two iterations of a
128-row loop (32 `i8` lanes of AVX2, unrolled four times) plus a remainder.
It's skipped under Miri, which doesn't run the vectorized code. With
`collect_overflowing` changed to drop the flag on odd rows, it fails at row 1.
> For the one-row edge checks, the current null mask also hides every signed
RHS `MAX` because the signed edge list has 12 entries and `MAX` is at index 11.
Right. With 12 edges, row `12x + y` is null exactly when `y % 3 == 2`. So
the array-array and one-row checks never paired a valid left value with a right
value of `MIN / 2`, `0`, `MAX / 2`, or `MAX`. `test_checked_integer_op` now
runs those checks with the mask and without it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]