comphead commented on code in PR #12062:
URL: https://github.com/apache/arrow-rs/pull/12062#discussion_r4634463156


##########
arrow-arith/src/numeric.rs:
##########
@@ -1812,6 +2164,335 @@ mod tests {
         assert_eq!(err, "Divide by zero error");
     }
 
+    type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+    type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+    fn assert_decimal_result<T: DecimalType>(

Review Comment:
   `assert_decimal_result` is the existing `assert_kernel_result` (line 1707) 
without the context string. `DecimalType` implies `ArrowPrimitiveType`, so 
these tests could call `assert_kernel_result::<T>`, and a context such as the 
row or the scalar would make a failure easier to locate.



##########
arrow/benches/arithmetic_kernels.rs:
##########
@@ -60,5 +67,160 @@ fn add_benchmark(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, add_benchmark);
+/// Like `create_primitive_array_range`, but with its own `seed`, so that two
+/// arrays hold different values and nulls.
+fn seeded_array<T>(

Review Comment:
   `seeded_array` is `create_primitive_array_range` with a seed, and #12058 
adds the same copy. Could it move to `arrow/src/util/bench_util.rs` as 
`create_primitive_array_range_with_seed`, next to 
`create_primitive_array_with_seed`? nit: `a64` and `b64` repeat the same 
expression, and a `decimal64 = |seed| ...` closure like `decimal32` would 
remove the repetition.



##########
arrow-arith/src/numeric.rs:
##########
@@ -1812,6 +2164,335 @@ mod tests {
         assert_eq!(err, "Divide by zero error");
     }
 
+    type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+    type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+    fn assert_decimal_result<T: DecimalType>(
+        actual: Result<ArrayRef, ArrowError>,
+        expected: Result<Vec<Option<T::Native>>, ArrowError>,
+    ) {
+        match (actual, expected) {
+            (Ok(actual), Ok(expected)) => {
+                let actual: Vec<_> = 
actual.as_primitive::<T>().iter().collect();
+                assert_eq!(actual, expected)
+            }
+            (Err(actual), Err(expected)) => assert_eq!(actual.to_string(), 
expected.to_string()),
+            (actual, expected) => panic!("got {actual:?}, expected 
{expected:?}"),
+        }
+    }
+
+    /// Checks `kernel` against `checked` for every pair of `edges`, with
+    /// arrays on both sides, with each pair as a one-row slice, and with a
+    /// scalar on either side. Every third row of the left array is null, so
+    /// null slots also hold values that overflow.
+    fn test_checked_decimal_op<T: DecimalType>(
+        edges: &[T::Native],
+        (l_scale, r_scale): (i8, i8),
+        kernel: Kernel,
+        checked: impl Fn(T::Native, T::Native) -> Result<T::Native, 
ArrowError>,
+    ) {
+        let decimal = |array: PrimitiveArray<T>, scale| {
+            array
+                .with_precision_and_scale(T::MAX_PRECISION, scale)
+                .unwrap()
+        };
+        let (l, r): (Vec<_>, Vec<_>) = edges
+            .iter()
+            .flat_map(|&x| edges.iter().map(move |&y| (x, y)))
+            .unzip();
+        let nulls = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 != 2));
+        let l_array = decimal(
+            PrimitiveArray::new(l.clone().into(), Some(nulls.clone())),
+            l_scale,
+        );
+        let r_array = 
decimal(PrimitiveArray::from_iter_values(r.iter().copied()), r_scale);
+        let row = |i: usize, x, y| match nulls.is_valid(i) {
+            true => checked(x, y).map(Some),
+            false => Ok(None),
+        };
+
+        // The kernels report the first valid row that overflows.
+        let expected = (0..l.len()).map(|i| row(i, l[i], r[i])).collect();
+        assert_decimal_result::<T>(kernel(&l_array, &r_array), expected);
+
+        // An overflow in one row sends the whole array to the checked kernel,
+        // which would hide a missed overflow in another row, so also check 
each
+        // row alone.
+        for i in 0..l.len() {
+            let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i, 1));
+            assert_decimal_result::<T>(actual, row(i, l[i], r[i]).map(|v| 
vec![v]));
+        }
+
+        for &x in edges {
+            let l_scalar = 
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), l_scale));
+            let expected = r.iter().map(|&y| checked(x, 
y).map(Some)).collect();
+            assert_decimal_result::<T>(kernel(&l_scalar, &r_array), expected);
+            let r_scalar = 
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), r_scale));
+            let expected = (0..l.len()).map(|i| row(i, l[i], x)).collect();
+            assert_decimal_result::<T>(kernel(&l_array, &r_scalar), expected);
+        }
+    }
+
+    /// Checks `add`, `sub` and `mul` of `T` at its edge values, with equal
+    /// scales and with either side rescaled by 100.
+    fn test_checked_decimal_ops<T: DecimalType>() {
+        let one = T::Native::ONE;
+        let three = T::Native::usize_as(3);
+        let two = T::Native::usize_as(2);
+        let hundred = T::Native::usize_as(100);
+        let (min, max) = (T::Native::MIN_TOTAL_ORDER, 
T::Native::MAX_TOTAL_ORDER);
+        let edges = [
+            min,
+            min.add_wrapping(one),
+            min.div_wrapping(two),
+            min.div_wrapping(hundred),
+            min.div_wrapping(hundred).sub_wrapping(one),
+            three.neg_wrapping(),
+            one.neg_wrapping(),
+            T::Native::ZERO,
+            one,
+            three,
+            max.div_wrapping(hundred),
+            max.div_wrapping(hundred).add_wrapping(one),
+            max.div_wrapping(two),
+            max.div_wrapping(two).add_wrapping(one),
+            max.sub_wrapping(one),
+            max,
+        ];
+
+        for scales in [(2, 2), (2, 4), (4, 2)] {
+            let (l_mul, r_mul) = match scales {
+                (2, 4) => (hundred, one),
+                (4, 2) => (one, hundred),
+                _ => (one, one),
+            };
+            let rescale = move |l: T::Native, r: T::Native| {
+                Ok::<_, ArrowError>((l.mul_checked(l_mul)?, 
r.mul_checked(r_mul)?))
+            };
+            test_checked_decimal_op::<T>(&edges, scales, add, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.add_checked(r)
+            });
+            test_checked_decimal_op::<T>(&edges, scales, sub, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.sub_checked(r)
+            });
+            test_checked_decimal_op::<T>(&edges, scales, add_wrapping, |l, r| {
+                let (l, r) = rescale(l, r)?;
+                l.add_checked(r)
+            });
+        }
+        // Multiplication doesn't rescale its operands.
+        test_checked_decimal_op::<T>(&edges, (2, 2), mul, |l, r| 
l.mul_checked(r));
+        test_checked_decimal_op::<T>(&edges, (2, 4), mul_wrapping, |l, r| 
l.mul_checked(r));
+    }
+
+    #[test]
+    fn test_checked_decimal_edges() {
+        test_checked_decimal_ops::<Decimal32Type>();
+        test_checked_decimal_ops::<Decimal64Type>();
+        test_checked_decimal_ops::<Decimal128Type>();
+        test_checked_decimal_ops::<Decimal256Type>();
+    }
+
+    #[test]
+    fn test_checked_decimal128_mul_i64_boundary() {
+        // `i128` multiplies through `i64` when both operands fit, so check 
both
+        // sides of each `i64` bound. `i64::MIN * i64::MIN` is 2^126.
+        let (min, max) = (i64::MIN as i128, i64::MAX as i128);
+        let edges = [min - 1, min, min + 1, -1, 0, 1, max - 1, max, max + 1];
+        test_checked_decimal_op::<Decimal128Type>(&edges, (2, 2), mul, |l, r| 
l.mul_checked(r));
+        test_checked_decimal_op::<Decimal128Type>(&edges, (2, 4), add, |l, r| {
+            l.mul_checked(100)?.add_checked(r)
+        });
+    }
+
+    /// Moves one row that overflows through a batch, so that each lane of the
+    /// vector loop and each row of the scalar remainder holds it once, with an
+    /// array and a scalar on either side. The rescaled cases overflow only in
+    /// the rescale. The same row is also checked as a null slot, which is not
+    /// an error.
+    fn test_checked_decimal_overflow_position<T: DecimalType>() {
+        // Three iterations of the widest vector loop we build, 8 `i32` lanes 
of
+        // AVX2 unrolled four times, plus a remainder.
+        const LEN: usize = 100;
+        let one = T::Native::ONE;
+        let two = T::Native::usize_as(2);
+        let hundred = T::Native::usize_as(100);
+        let (min, max) = (T::Native::MIN_TOTAL_ORDER, 
T::Native::MAX_TOTAL_ORDER);
+        // Multiplying by 100 overflows, but adding or subtracting 1 doesn't.
+        let rescale_overflow = max.div_wrapping(hundred).add_wrapping(one);
+        // The kernel, its checked scalar operation, whether it rescales, the
+        // scales of the batch and the other operand, the value that overflows,
+        // and the other operand.
+        type Case<N> = (Kernel, Checked<N>, bool, (i8, i8), N, N);
+        let cases: [Case<T::Native>; 5] = [
+            (add, |l, r| l.add_checked(r), true, (2, 2), max, one),
+            (sub, |l, r| l.sub_checked(r), true, (2, 2), min, one),
+            (mul, |l, r| l.mul_checked(r), false, (2, 2), max, two),
+            (
+                add,
+                |l, r| l.add_checked(r),
+                true,
+                (2, 4),
+                rescale_overflow,
+                one,
+            ),
+            (
+                sub,
+                |l, r| l.sub_checked(r),
+                true,
+                (2, 4),
+                rescale_overflow,
+                one,
+            ),
+        ];
+        let decimal = |array: PrimitiveArray<T>, scale| {
+            array
+                .with_precision_and_scale(T::MAX_PRECISION, scale)
+                .unwrap()
+        };
+        for (kernel, checked, rescales, (batch_scale, other_scale), edge, 
other) in cases {
+            // The checked result of `l op r`, rescaling the operand with the 
smaller scale.
+            let row = |l: T::Native, l_scale, r: T::Native, r_scale| {
+                if !rescales {
+                    return checked(l, r);
+                }
+                let l = if l_scale < r_scale {
+                    l.mul_checked(hundred)?
+                } else {
+                    l
+                };
+                let r = if r_scale < l_scale {
+                    r.mul_checked(hundred)?
+                } else {
+                    r
+                };
+                checked(l, r)
+            };
+            let other_array = decimal(PrimitiveArray::from_value(other, LEN), 
other_scale);
+            let other_scalar =
+                Scalar::new(decimal(PrimitiveArray::from_value(other, 1), 
other_scale));
+            for i in 0..LEN {
+                let mut values = vec![one; LEN];
+                values[i] = edge;
+                for null in [None, Some(i)] {
+                    let nulls = null.map(|i| 
NullBuffer::from_iter((0..LEN).map(|j| j != i)));
+                    let array = decimal(
+                        PrimitiveArray::new(values.clone().into(), nulls),
+                        batch_scale,
+                    );
+                    let expected =
+                        |batch_left: bool| -> Result<Vec<Option<T::Native>>, 
ArrowError> {
+                            (0..LEN)
+                                .map(|j| match (Some(j) == null, batch_left) {
+                                    (true, _) => Ok(None),
+                                    (false, true) => {
+                                        row(values[j], batch_scale, other, 
other_scale).map(Some)
+                                    }
+                                    (false, false) => {
+                                        row(other, other_scale, values[j], 
batch_scale).map(Some)
+                                    }
+                                })
+                                .collect()
+                        };
+                    assert_decimal_result::<T>(kernel(&array, &other_array), 
expected(true));
+                    assert_decimal_result::<T>(kernel(&other_array, &array), 
expected(false));
+                    assert_decimal_result::<T>(kernel(&array, &other_scalar), 
expected(true));
+                    assert_decimal_result::<T>(kernel(&other_scalar, &array), 
expected(false));
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_checked_decimal_null_slot_overflow_sliced() {

Review Comment:
   After #12058 lands, this test runs the same `overflowing_op` and `any_valid` 
path as `test_checked_null_slot_overflow_sliced` there, because equal-scale 
`Decimal64` add goes straight to `overflowing_op`. The only difference is `i64` 
instead of `i32`. Could it be dropped when rebasing? The `Kernel` and `Checked` 
aliases would also collide, and `test_checked_decimal_op` and 
`test_checked_integer_op` are close enough that one helper taking a closure for 
the data type could serve both.



##########
arrow-arith/src/numeric.rs:
##########
@@ -1192,31 +1500,72 @@ where
             let l_mul = T::Native::usize_as(10).pow_checked((result_scale - 
s1) as _)?;
             let r_mul = T::Native::usize_as(10).pow_checked((result_scale - 
s2) as _)?;
 
+            // A scalar with the smaller scale is rescaled once here, rather 
than
+            // on every row, which makes its multiplier one.
+            let (l_rescaled, r_rescaled) =
+                (rescale_scalar(l, l_s, l_mul), rescale_scalar(r, r_s, r_mul));
+            let one = T::Native::ONE;
+            let (l, l_mul) = l_rescaled.as_ref().map_or((l, l_mul), |l| (l, 
one));
+            let (r, r_mul) = r_rescaled.as_ref().map_or((r, r_mul), |r| (r, 
one));
+            let unscaled = l_mul.is_eq(one) && r_mul.is_eq(one);
+
             match op {
-                // Equal scales make both decimal multipliers one.
-                Op::Add | Op::AddWrapping if s1 == s2 => {
-                    try_op!(l, l_s, r, r_s, l.add_checked(r))
+                Op::Add | Op::AddWrapping if unscaled => {
+                    match overflowing_op(l, l_s, r, r_s, 
T::Native::add_overflowing)? {
+                        Some(array) => array,
+                        None => try_op!(l, l_s, r, r_s, l.add_checked(r)),
+                    }
                 }
-                Op::Sub | Op::SubWrapping if s1 == s2 => {
-                    try_op!(l, l_s, r, r_s, l.sub_checked(r))
+                Op::Sub | Op::SubWrapping if unscaled => {
+                    match overflowing_op(l, l_s, r, r_s, 
T::Native::sub_overflowing)? {
+                        Some(array) => array,
+                        None => try_op!(l, l_s, r, r_s, l.sub_checked(r)),
+                    }
                 }
+                // `move` copies the multipliers into the closures. Borrowed, 
they
+                // keep the loop from vectorizing under thin LTO, like the 
borrowed
+                // captures #11525 fixed in `unary`.
                 Op::Add | Op::AddWrapping => {
-                    try_op!(
-                        l,
-                        l_s,
-                        r,
-                        r_s,
-                        
l.mul_checked(l_mul)?.add_checked(r.mul_checked(r_mul)?)
-                    )
+                    let add = T::Native::add_overflowing;

Review Comment:
   nit: the rescaled `Add` and `Sub` arms differ only in `add`/`sub` and 
`add_checked`/`sub_checked`. A local macro, as `try_op!` does for the scalar 
dispatch, would keep the two in one place. Also, `rescale_scalar` sits between 
`rescale_left` and `rescale_right`, so the two rescale helpers would read 
better next to each other.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to