comphead commented on code in PR #12062:
URL: https://github.com/apache/arrow-rs/pull/12062#discussion_r4634463137
##########
arrow/benches/arithmetic_kernels.rs:
##########
@@ -60,5 +67,160 @@ fn add_benchmark(c: &mut Criterion) {
}
}
-criterion_group!(benches, add_benchmark);
+/// Like `create_primitive_array_range`, but with its own `seed`, so that two
+/// arrays hold different values and nulls.
+fn seeded_array<T>(
Review Comment:
`seeded_array` is `create_primitive_array_range` with a seed, and
https://github.com/apache/arrow-rs/pull/12058 adds the same copy. Could it move
to `arrow/src/util/bench_util.rs` as `create_primitive_array_range_with_seed`,
next to `create_primitive_array_with_seed`? nit: `a64` and `b64` repeat the
same expression, and a `decimal64 = |seed| ...` closure like `decimal32` would
remove the repetition.
##########
arrow-arith/src/numeric.rs:
##########
@@ -1812,6 +2164,335 @@ mod tests {
assert_eq!(err, "Divide by zero error");
}
+ type Kernel = fn(&dyn Datum, &dyn Datum) -> Result<ArrayRef, ArrowError>;
+ type Checked<N> = fn(N, N) -> Result<N, ArrowError>;
+
+ fn assert_decimal_result<T: DecimalType>(
+ actual: Result<ArrayRef, ArrowError>,
+ expected: Result<Vec<Option<T::Native>>, ArrowError>,
+ ) {
+ match (actual, expected) {
+ (Ok(actual), Ok(expected)) => {
+ let actual: Vec<_> =
actual.as_primitive::<T>().iter().collect();
+ assert_eq!(actual, expected)
+ }
+ (Err(actual), Err(expected)) => assert_eq!(actual.to_string(),
expected.to_string()),
+ (actual, expected) => panic!("got {actual:?}, expected
{expected:?}"),
+ }
+ }
+
+ /// Checks `kernel` against `checked` for every pair of `edges`, with
+ /// arrays on both sides, with each pair as a one-row slice, and with a
+ /// scalar on either side. Every third row of the left array is null, so
+ /// null slots also hold values that overflow.
+ fn test_checked_decimal_op<T: DecimalType>(
+ edges: &[T::Native],
+ (l_scale, r_scale): (i8, i8),
+ kernel: Kernel,
+ checked: impl Fn(T::Native, T::Native) -> Result<T::Native,
ArrowError>,
+ ) {
+ let decimal = |array: PrimitiveArray<T>, scale| {
+ array
+ .with_precision_and_scale(T::MAX_PRECISION, scale)
+ .unwrap()
+ };
+ let (l, r): (Vec<_>, Vec<_>) = edges
+ .iter()
+ .flat_map(|&x| edges.iter().map(move |&y| (x, y)))
+ .unzip();
+ let nulls = NullBuffer::from_iter((0..l.len()).map(|i| i % 3 != 2));
+ let l_array = decimal(
+ PrimitiveArray::new(l.clone().into(), Some(nulls.clone())),
+ l_scale,
+ );
+ let r_array =
decimal(PrimitiveArray::from_iter_values(r.iter().copied()), r_scale);
+ let row = |i: usize, x, y| match nulls.is_valid(i) {
+ true => checked(x, y).map(Some),
+ false => Ok(None),
+ };
+
+ // The kernels report the first valid row that overflows.
+ let expected = (0..l.len()).map(|i| row(i, l[i], r[i])).collect();
+ assert_decimal_result::<T>(kernel(&l_array, &r_array), expected);
+
+ // An overflow in one row sends the whole array to the checked kernel,
+ // which would hide a missed overflow in another row, so also check
each
+ // row alone.
+ for i in 0..l.len() {
+ let actual = kernel(&l_array.slice(i, 1), &r_array.slice(i, 1));
+ assert_decimal_result::<T>(actual, row(i, l[i], r[i]).map(|v|
vec![v]));
+ }
+
+ for &x in edges {
+ let l_scalar =
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), l_scale));
+ let expected = r.iter().map(|&y| checked(x,
y).map(Some)).collect();
+ assert_decimal_result::<T>(kernel(&l_scalar, &r_array), expected);
+ let r_scalar =
Scalar::new(decimal(PrimitiveArray::from_iter_values([x]), r_scale));
+ let expected = (0..l.len()).map(|i| row(i, l[i], x)).collect();
+ assert_decimal_result::<T>(kernel(&l_array, &r_scalar), expected);
+ }
+ }
+
+ /// Checks `add`, `sub` and `mul` of `T` at its edge values, with equal
+ /// scales and with either side rescaled by 100.
+ fn test_checked_decimal_ops<T: DecimalType>() {
+ let one = T::Native::ONE;
+ let three = T::Native::usize_as(3);
+ let two = T::Native::usize_as(2);
+ let hundred = T::Native::usize_as(100);
+ let (min, max) = (T::Native::MIN_TOTAL_ORDER,
T::Native::MAX_TOTAL_ORDER);
+ let edges = [
+ min,
+ min.add_wrapping(one),
+ min.div_wrapping(two),
+ min.div_wrapping(hundred),
+ min.div_wrapping(hundred).sub_wrapping(one),
+ three.neg_wrapping(),
+ one.neg_wrapping(),
+ T::Native::ZERO,
+ one,
+ three,
+ max.div_wrapping(hundred),
+ max.div_wrapping(hundred).add_wrapping(one),
+ max.div_wrapping(two),
+ max.div_wrapping(two).add_wrapping(one),
+ max.sub_wrapping(one),
+ max,
+ ];
+
+ for scales in [(2, 2), (2, 4), (4, 2)] {
+ let (l_mul, r_mul) = match scales {
+ (2, 4) => (hundred, one),
+ (4, 2) => (one, hundred),
+ _ => (one, one),
+ };
+ let rescale = move |l: T::Native, r: T::Native| {
+ Ok::<_, ArrowError>((l.mul_checked(l_mul)?,
r.mul_checked(r_mul)?))
+ };
+ test_checked_decimal_op::<T>(&edges, scales, add, |l, r| {
+ let (l, r) = rescale(l, r)?;
+ l.add_checked(r)
+ });
+ test_checked_decimal_op::<T>(&edges, scales, sub, |l, r| {
+ let (l, r) = rescale(l, r)?;
+ l.sub_checked(r)
+ });
+ test_checked_decimal_op::<T>(&edges, scales, add_wrapping, |l, r| {
+ let (l, r) = rescale(l, r)?;
+ l.add_checked(r)
+ });
+ }
+ // Multiplication doesn't rescale its operands.
+ test_checked_decimal_op::<T>(&edges, (2, 2), mul, |l, r|
l.mul_checked(r));
+ test_checked_decimal_op::<T>(&edges, (2, 4), mul_wrapping, |l, r|
l.mul_checked(r));
+ }
+
+ #[test]
+ fn test_checked_decimal_edges() {
+ test_checked_decimal_ops::<Decimal32Type>();
+ test_checked_decimal_ops::<Decimal64Type>();
+ test_checked_decimal_ops::<Decimal128Type>();
+ test_checked_decimal_ops::<Decimal256Type>();
+ }
+
+ #[test]
+ fn test_checked_decimal128_mul_i64_boundary() {
+ // `i128` multiplies through `i64` when both operands fit, so check
both
+ // sides of each `i64` bound. `i64::MIN * i64::MIN` is 2^126.
+ let (min, max) = (i64::MIN as i128, i64::MAX as i128);
+ let edges = [min - 1, min, min + 1, -1, 0, 1, max - 1, max, max + 1];
+ test_checked_decimal_op::<Decimal128Type>(&edges, (2, 2), mul, |l, r|
l.mul_checked(r));
+ test_checked_decimal_op::<Decimal128Type>(&edges, (2, 4), add, |l, r| {
+ l.mul_checked(100)?.add_checked(r)
+ });
+ }
+
+ /// Moves one row that overflows through a batch, so that each lane of the
+ /// vector loop and each row of the scalar remainder holds it once, with an
+ /// array and a scalar on either side. The rescaled cases overflow only in
+ /// the rescale. The same row is also checked as a null slot, which is not
+ /// an error.
+ fn test_checked_decimal_overflow_position<T: DecimalType>() {
+ // Three iterations of the widest vector loop we build, 8 `i32` lanes
of
+ // AVX2 unrolled four times, plus a remainder.
+ const LEN: usize = 100;
+ let one = T::Native::ONE;
+ let two = T::Native::usize_as(2);
+ let hundred = T::Native::usize_as(100);
+ let (min, max) = (T::Native::MIN_TOTAL_ORDER,
T::Native::MAX_TOTAL_ORDER);
+ // Multiplying by 100 overflows, but adding or subtracting 1 doesn't.
+ let rescale_overflow = max.div_wrapping(hundred).add_wrapping(one);
+ // The kernel, its checked scalar operation, whether it rescales, the
+ // scales of the batch and the other operand, the value that overflows,
+ // and the other operand.
+ type Case<N> = (Kernel, Checked<N>, bool, (i8, i8), N, N);
+ let cases: [Case<T::Native>; 5] = [
+ (add, |l, r| l.add_checked(r), true, (2, 2), max, one),
+ (sub, |l, r| l.sub_checked(r), true, (2, 2), min, one),
+ (mul, |l, r| l.mul_checked(r), false, (2, 2), max, two),
+ (
+ add,
+ |l, r| l.add_checked(r),
+ true,
+ (2, 4),
+ rescale_overflow,
+ one,
+ ),
+ (
+ sub,
+ |l, r| l.sub_checked(r),
+ true,
+ (2, 4),
+ rescale_overflow,
+ one,
+ ),
+ ];
+ let decimal = |array: PrimitiveArray<T>, scale| {
+ array
+ .with_precision_and_scale(T::MAX_PRECISION, scale)
+ .unwrap()
+ };
+ for (kernel, checked, rescales, (batch_scale, other_scale), edge,
other) in cases {
+ // The checked result of `l op r`, rescaling the operand with the
smaller scale.
+ let row = |l: T::Native, l_scale, r: T::Native, r_scale| {
+ if !rescales {
+ return checked(l, r);
+ }
+ let l = if l_scale < r_scale {
+ l.mul_checked(hundred)?
+ } else {
+ l
+ };
+ let r = if r_scale < l_scale {
+ r.mul_checked(hundred)?
+ } else {
+ r
+ };
+ checked(l, r)
+ };
+ let other_array = decimal(PrimitiveArray::from_value(other, LEN),
other_scale);
+ let other_scalar =
+ Scalar::new(decimal(PrimitiveArray::from_value(other, 1),
other_scale));
+ for i in 0..LEN {
+ let mut values = vec![one; LEN];
+ values[i] = edge;
+ for null in [None, Some(i)] {
+ let nulls = null.map(|i|
NullBuffer::from_iter((0..LEN).map(|j| j != i)));
+ let array = decimal(
+ PrimitiveArray::new(values.clone().into(), nulls),
+ batch_scale,
+ );
+ let expected =
+ |batch_left: bool| -> Result<Vec<Option<T::Native>>,
ArrowError> {
+ (0..LEN)
+ .map(|j| match (Some(j) == null, batch_left) {
+ (true, _) => Ok(None),
+ (false, true) => {
+ row(values[j], batch_scale, other,
other_scale).map(Some)
+ }
+ (false, false) => {
+ row(other, other_scale, values[j],
batch_scale).map(Some)
+ }
+ })
+ .collect()
+ };
+ assert_decimal_result::<T>(kernel(&array, &other_array),
expected(true));
+ assert_decimal_result::<T>(kernel(&other_array, &array),
expected(false));
+ assert_decimal_result::<T>(kernel(&array, &other_scalar),
expected(true));
+ assert_decimal_result::<T>(kernel(&other_scalar, &array),
expected(false));
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn test_checked_decimal_null_slot_overflow_sliced() {
Review Comment:
After https://github.com/apache/arrow-rs/pull/12058 lands, this test runs
the same `overflowing_op` and `any_valid` path as
`test_checked_null_slot_overflow_sliced` there, because equal-scale `Decimal64`
add goes straight to `overflowing_op`. The only difference is `i64` instead of
`i32`. Could it be dropped when rebasing? The `Kernel` and `Checked` aliases
would also collide, and `test_checked_decimal_op` and `test_checked_integer_op`
are close enough that one helper taking a closure for the data type could serve
both.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]