This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11078-c0fcd708bfa863ba18757b3ebb2f96a904bb899d in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 0e4a50115584adee2852e33eadb68d0a0ccd4094 Author: Anand Hegde <[email protected]> AuthorDate: Tue Oct 27 00:18:31 2026 +0000 Move remaining decimal adapters into the decimal submodule (#11078) # Which issue does this PR close? - Part of #11032. # Rationale for this change Continues splitting up `cast/mod.rs`. # What changes are included in this PR? Moves the remaining decimal adapters (`integer_to_decimal_native`, `integers_fit_digits`, `cast_integer_to_decimal`, `cast_from_decimal`, `cast_to_decimal`) into the existing `cast/decimal.rs`. Code is unchanged apart from `pub(crate)` on `cast_from_decimal` and `cast_to_decimal`. # Are these changes tested? Covered by existing tests. # Are there any significant user-facing changes? No. --- arrow-cast/src/cast/decimal.rs | 284 +++++++++++++++++++++++++++++++++++++++++ arrow-cast/src/cast/mod.rs | 284 ----------------------------------------- 2 files changed, 284 insertions(+), 284 deletions(-) diff --git a/arrow-cast/src/cast/decimal.rs b/arrow-cast/src/cast/decimal.rs index 367f3207df..99b0905dc0 100644 --- a/arrow-cast/src/cast/decimal.rs +++ b/arrow-cast/src/cast/decimal.rs @@ -911,6 +911,290 @@ where Ok(Arc::new(array)) } +/// Convert an integer to a decimal native value without wrapping. +/// +/// `AsPrimitive` / `as` silently truncates when the source is wider than `M` +/// (for example `5_000_000_000i64 as i32`). All integer sources fit in `i128` +/// losslessly, which [`DecimalCast`] then converts to the decimal native type +/// with a range check. For types that always fit (e.g. `i64` to `Decimal128`) this +/// should get optimized to being equivalent to `i64 as i128`. +fn integer_to_decimal_native<I, M>(value: I) -> Option<M> +where + I: Into<i128>, + M: DecimalCast, +{ + M::from_decimal(value.into()) +} + +/// Returns true if every value of `array` has at most `digits` decimal digits. +/// +/// If the source type cannot hold that many digits, every value fits; +/// otherwise the data is scanned, including null slots, since `unary` +/// converts those as well. +fn integers_fit_digits<T>(array: &PrimitiveArray<T>, digits: u32) -> bool +where + T: ArrowPrimitiveType, + T::Native: ArrowNativeTypeOp, +{ + let Ok(bound) = T::Native::usize_as(10).pow_checked(digits) else { + // The type's maximum is below the bound, and its minimum, one larger + // in magnitude, is a power of two that no power of ten can equal. + return true; + }; + let hi = bound.sub_wrapping(T::Native::ONE); + // Unsigned types have no negative values to bound. + let lo = T::Native::ZERO.sub_checked(hi).unwrap_or(T::Native::ZERO); + + // The scan works in chunks, so an array that does not fit is rejected after + // its first failing chunk. Each chunk is checked without branching, so it + // vectorizes. + array + .values() + .chunks(64) + .all(|chunk| chunk.iter().fold(true, |ok, &v| ok & (v >= lo) & (v <= hi))) +} + +fn cast_integer_to_decimal< + T: ArrowPrimitiveType, + D: DecimalType + ArrowPrimitiveType<Native = M>, + M, +>( + array: &PrimitiveArray<T>, + precision: u8, + scale: i8, + cast_options: &CastOptions, +) -> Result<ArrayRef, ArrowError> +where + <T as ArrowPrimitiveType>::Native: ArrowNativeTypeOp + Into<i128>, + M: ArrowNativeTypeOp + DecimalCast, +{ + let overflow = |v: T::Native| { + ArrowError::CastError(format!( + "Cannot cast to {}({precision}, {scale}). Overflowing on {v:?}", + D::PREFIX, + )) + }; + + // An invalid precision or scale would let the bound below admit values the + // decimal native type cannot hold, so reject it before choosing a kernel. + validate_decimal_precision_and_scale::<D>(precision, scale)?; + + // Scan for values that don't fit in the target type before choosing a + // kernel: in the common case that all values fit, the cast is infallible + // and we can use the `unary` kernel, which is very fast. Empirically, the + // cost of doing the out-of-range scan is small relative to the win from + // using a faster kernel. + // + // A value fits when it has at most `precision - scale` decimal digits: a + // nonnegative scale appends that many trailing zeros, and a negative scale + // removes digits first. Validation guarantees the difference is not + // negative. + let digits = (precision as i32 - scale as i32) as u32; + let all_fit = integers_fit_digits(array, digits); + + let array = if scale < 0 { + // Compute the scale factor once in the source type. Scaling before the + // checked conversion permits values that only fit the decimal native + // type after scaling. + // + // scale_factor is always > 0, so div_wrapping(scale_factor) can never + // overflow/panic + let scale_factor = T::Native::usize_as(10) + .pow_checked(scale.unsigned_abs() as u32) + .ok(); + + match (scale_factor, cast_options.safe) { + (Some(scale_factor), _) if all_fit => array.unary::<_, D>(|v| { + integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor)) + .expect("value fits the decimal") + }), + (Some(scale_factor), true) => array.unary_opt::<_, D>(|v| { + let v = integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor))?; + (D::is_valid_decimal_precision(v, precision)).then_some(v) + }), + (Some(scale_factor), false) => array.try_unary::<_, D, _>(|v| { + let v = integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor)) + .ok_or_else(|| overflow(v))?; + D::validate_decimal_precision(v, precision, scale).map(|()| v) + })?, + // A scale factor that overflows the source type is larger than all + // source values, so integer division produces zero. + // + // For a well formed decimal scale, this path should never be reachable. + (None, _) => array.unary::<_, D>(|_| M::ZERO), + } + } else { + let scale_factor = M::usize_as(10) + .pow_checked(scale.unsigned_abs() as u32) + .map_err(|_| { + ArrowError::CastError(format!( + "Cannot cast to {:?}({}, {}). The scale causes overflow.", + D::PREFIX, + precision, + scale, + )) + })?; + + if all_fit { + array.unary::<_, D>(|v| { + integer_to_decimal_native::<_, M>(v) + .expect("value fits the decimal") + .mul_wrapping(scale_factor) + }) + } else if cast_options.safe { + array.unary_opt::<_, D>(|v| { + let v = integer_to_decimal_native::<_, M>(v) + .and_then(|v| v.mul_checked(scale_factor).ok())?; + (D::is_valid_decimal_precision(v, precision)).then_some(v) + }) + } else { + array.try_unary::<_, D, _>(|v| { + let v = integer_to_decimal_native::<_, M>(v) + .ok_or_else(|| overflow(v)) + .and_then(|v| v.mul_checked(scale_factor))?; + D::validate_decimal_precision(v, precision, scale).map(|()| v) + })? + } + }; + + Ok(Arc::new(array.with_precision_and_scale(precision, scale)?)) +} + +pub(crate) fn cast_from_decimal<D, F>( + array: &dyn Array, + scale: &i8, + from_type: &DataType, + to_type: &DataType, + as_float: F, + cast_options: &CastOptions, +) -> Result<ArrayRef, ArrowError> +where + D: DecimalType + ArrowPrimitiveType, + <D as ArrowPrimitiveType>::Native: ToPrimitive, + F: Fn(D::Native) -> f64, +{ + use DataType::*; + // cast decimal to other type + match to_type { + UInt8 => cast_decimal_to_integer::<D, UInt8Type>(array, *scale, cast_options), + UInt16 => cast_decimal_to_integer::<D, UInt16Type>(array, *scale, cast_options), + UInt32 => cast_decimal_to_integer::<D, UInt32Type>(array, *scale, cast_options), + UInt64 => cast_decimal_to_integer::<D, UInt64Type>(array, *scale, cast_options), + Int8 => cast_decimal_to_integer::<D, Int8Type>(array, *scale, cast_options), + Int16 => cast_decimal_to_integer::<D, Int16Type>(array, *scale, cast_options), + Int32 => cast_decimal_to_integer::<D, Int32Type>(array, *scale, cast_options), + Int64 => cast_decimal_to_integer::<D, Int64Type>(array, *scale, cast_options), + Float16 => cast_decimal_to_float::<D, Float16Type, _>(array, |x| { + half::f16::from_f64(single_decimal_to_float_lossy::<D, F>( + &as_float, + x, + <i32 as From<i8>>::from(*scale), + )) + }), + Float32 => cast_decimal_to_f32::<D, F>(array, &as_float, *scale), + Float64 => cast_decimal_to_f64::<D, F>(array, &as_float, *scale), + Utf8View => value_to_string_view(array, cast_options), + Utf8 => value_to_string::<i32>(array, cast_options), + LargeUtf8 => value_to_string::<i64>(array, cast_options), + Null => Ok(new_null_array(to_type, array.len())), + _ => Err(ArrowError::CastError(format!( + "Casting from {from_type} to {to_type} not supported" + ))), + } +} + +pub(crate) fn cast_to_decimal<D, M>( + array: &dyn Array, + precision: &u8, + scale: &i8, + from_type: &DataType, + to_type: &DataType, + cast_options: &CastOptions, +) -> Result<ArrayRef, ArrowError> +where + D: DecimalType + ArrowPrimitiveType<Native = M>, + M: ArrowNativeTypeOp + DecimalCast, +{ + use DataType::*; + // cast data to decimal + match from_type { + UInt8 => cast_integer_to_decimal::<_, D, M>( + array.as_primitive::<UInt8Type>(), + *precision, + *scale, + cast_options, + ), + UInt16 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<UInt16Type>(), + *precision, + *scale, + cast_options, + ), + UInt32 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<UInt32Type>(), + *precision, + *scale, + cast_options, + ), + UInt64 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<UInt64Type>(), + *precision, + *scale, + cast_options, + ), + Int8 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<Int8Type>(), + *precision, + *scale, + cast_options, + ), + Int16 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<Int16Type>(), + *precision, + *scale, + cast_options, + ), + Int32 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<Int32Type>(), + *precision, + *scale, + cast_options, + ), + Int64 => cast_integer_to_decimal::<_, D, _>( + array.as_primitive::<Int64Type>(), + *precision, + *scale, + cast_options, + ), + Float16 => cast_floating_point_to_decimal::<_, D>( + array.as_primitive::<Float16Type>(), + *precision, + *scale, + cast_options, + ), + Float32 => cast_floating_point_to_decimal::<_, D>( + array.as_primitive::<Float32Type>(), + *precision, + *scale, + cast_options, + ), + Float64 => cast_floating_point_to_decimal::<_, D>( + array.as_primitive::<Float64Type>(), + *precision, + *scale, + cast_options, + ), + Utf8View | Utf8 => { + cast_string_to_decimal::<D, i32>(array, *precision, *scale, cast_options) + } + LargeUtf8 => cast_string_to_decimal::<D, i64>(array, *precision, *scale, cast_options), + Null => Ok(new_null_array(to_type, array.len())), + _ => Err(ArrowError::CastError(format!( + "Casting from {from_type} to {to_type} not supported" + ))), + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/arrow-cast/src/cast/mod.rs b/arrow-cast/src/cast/mod.rs index 25fbf0680b..eab086cabe 100644 --- a/arrow-cast/src/cast/mod.rs +++ b/arrow-cast/src/cast/mod.rs @@ -479,155 +479,6 @@ pub fn cast(array: &dyn Array, to_type: &DataType) -> Result<ArrayRef, ArrowErro cast_with_options(array, to_type, &CastOptions::default()) } -/// Convert an integer to a decimal native value without wrapping. -/// -/// `AsPrimitive` / `as` silently truncates when the source is wider than `M` -/// (for example `5_000_000_000i64 as i32`). All integer sources fit in `i128` -/// losslessly, which [`DecimalCast`] then converts to the decimal native type -/// with a range check. For types that always fit (e.g. `i64` to `Decimal128`) this -/// should get optimized to being equivalent to `i64 as i128`. -fn integer_to_decimal_native<I, M>(value: I) -> Option<M> -where - I: Into<i128>, - M: DecimalCast, -{ - M::from_decimal(value.into()) -} - -/// Returns true if every value of `array` has at most `digits` decimal digits. -/// -/// If the source type cannot hold that many digits, every value fits; -/// otherwise the data is scanned, including null slots, since `unary` -/// converts those as well. -fn integers_fit_digits<T>(array: &PrimitiveArray<T>, digits: u32) -> bool -where - T: ArrowPrimitiveType, - T::Native: ArrowNativeTypeOp, -{ - let Ok(bound) = T::Native::usize_as(10).pow_checked(digits) else { - // The type's maximum is below the bound, and its minimum, one larger - // in magnitude, is a power of two that no power of ten can equal. - return true; - }; - let hi = bound.sub_wrapping(T::Native::ONE); - // Unsigned types have no negative values to bound. - let lo = T::Native::ZERO.sub_checked(hi).unwrap_or(T::Native::ZERO); - - // The scan works in chunks, so an array that does not fit is rejected after - // its first failing chunk. Each chunk is checked without branching, so it - // vectorizes. - array - .values() - .chunks(64) - .all(|chunk| chunk.iter().fold(true, |ok, &v| ok & (v >= lo) & (v <= hi))) -} - -fn cast_integer_to_decimal< - T: ArrowPrimitiveType, - D: DecimalType + ArrowPrimitiveType<Native = M>, - M, ->( - array: &PrimitiveArray<T>, - precision: u8, - scale: i8, - cast_options: &CastOptions, -) -> Result<ArrayRef, ArrowError> -where - <T as ArrowPrimitiveType>::Native: ArrowNativeTypeOp + Into<i128>, - M: ArrowNativeTypeOp + DecimalCast, -{ - let overflow = |v: T::Native| { - ArrowError::CastError(format!( - "Cannot cast to {}({precision}, {scale}). Overflowing on {v:?}", - D::PREFIX, - )) - }; - - // An invalid precision or scale would let the bound below admit values the - // decimal native type cannot hold, so reject it before choosing a kernel. - validate_decimal_precision_and_scale::<D>(precision, scale)?; - - // Scan for values that don't fit in the target type before choosing a - // kernel: in the common case that all values fit, the cast is infallible - // and we can use the `unary` kernel, which is very fast. Empirically, the - // cost of doing the out-of-range scan is small relative to the win from - // using a faster kernel. - // - // A value fits when it has at most `precision - scale` decimal digits: a - // nonnegative scale appends that many trailing zeros, and a negative scale - // removes digits first. Validation guarantees the difference is not - // negative. - let digits = (precision as i32 - scale as i32) as u32; - let all_fit = integers_fit_digits(array, digits); - - let array = if scale < 0 { - // Compute the scale factor once in the source type. Scaling before the - // checked conversion permits values that only fit the decimal native - // type after scaling. - // - // scale_factor is always > 0, so div_wrapping(scale_factor) can never - // overflow/panic - let scale_factor = T::Native::usize_as(10) - .pow_checked(scale.unsigned_abs() as u32) - .ok(); - - match (scale_factor, cast_options.safe) { - (Some(scale_factor), _) if all_fit => array.unary::<_, D>(|v| { - integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor)) - .expect("value fits the decimal") - }), - (Some(scale_factor), true) => array.unary_opt::<_, D>(|v| { - let v = integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor))?; - (D::is_valid_decimal_precision(v, precision)).then_some(v) - }), - (Some(scale_factor), false) => array.try_unary::<_, D, _>(|v| { - let v = integer_to_decimal_native::<_, M>(v.div_wrapping(scale_factor)) - .ok_or_else(|| overflow(v))?; - D::validate_decimal_precision(v, precision, scale).map(|()| v) - })?, - // A scale factor that overflows the source type is larger than all - // source values, so integer division produces zero. - // - // For a well formed decimal scale, this path should never be reachable. - (None, _) => array.unary::<_, D>(|_| M::ZERO), - } - } else { - let scale_factor = M::usize_as(10) - .pow_checked(scale.unsigned_abs() as u32) - .map_err(|_| { - ArrowError::CastError(format!( - "Cannot cast to {:?}({}, {}). The scale causes overflow.", - D::PREFIX, - precision, - scale, - )) - })?; - - if all_fit { - array.unary::<_, D>(|v| { - integer_to_decimal_native::<_, M>(v) - .expect("value fits the decimal") - .mul_wrapping(scale_factor) - }) - } else if cast_options.safe { - array.unary_opt::<_, D>(|v| { - let v = integer_to_decimal_native::<_, M>(v) - .and_then(|v| v.mul_checked(scale_factor).ok())?; - (D::is_valid_decimal_precision(v, precision)).then_some(v) - }) - } else { - array.try_unary::<_, D, _>(|v| { - let v = integer_to_decimal_native::<_, M>(v) - .ok_or_else(|| overflow(v)) - .and_then(|v| v.mul_checked(scale_factor))?; - D::validate_decimal_precision(v, precision, scale).map(|()| v) - })? - } - }; - - Ok(Arc::new(array.with_precision_and_scale(precision, scale)?)) -} - /// Cast the primitive array using [`PrimitiveArray::reinterpret_cast`] fn cast_reinterpret_arrays<I: ArrowPrimitiveType, O: ArrowPrimitiveType<Native = I::Native>>( array: &dyn Array, @@ -2129,141 +1980,6 @@ pub fn cast_with_options( } } -fn cast_from_decimal<D, F>( - array: &dyn Array, - scale: &i8, - from_type: &DataType, - to_type: &DataType, - as_float: F, - cast_options: &CastOptions, -) -> Result<ArrayRef, ArrowError> -where - D: DecimalType + ArrowPrimitiveType, - <D as ArrowPrimitiveType>::Native: ToPrimitive, - F: Fn(D::Native) -> f64, -{ - use DataType::*; - // cast decimal to other type - match to_type { - UInt8 => cast_decimal_to_integer::<D, UInt8Type>(array, *scale, cast_options), - UInt16 => cast_decimal_to_integer::<D, UInt16Type>(array, *scale, cast_options), - UInt32 => cast_decimal_to_integer::<D, UInt32Type>(array, *scale, cast_options), - UInt64 => cast_decimal_to_integer::<D, UInt64Type>(array, *scale, cast_options), - Int8 => cast_decimal_to_integer::<D, Int8Type>(array, *scale, cast_options), - Int16 => cast_decimal_to_integer::<D, Int16Type>(array, *scale, cast_options), - Int32 => cast_decimal_to_integer::<D, Int32Type>(array, *scale, cast_options), - Int64 => cast_decimal_to_integer::<D, Int64Type>(array, *scale, cast_options), - Float16 => cast_decimal_to_float::<D, Float16Type, _>(array, |x| { - half::f16::from_f64(single_decimal_to_float_lossy::<D, F>( - &as_float, - x, - <i32 as From<i8>>::from(*scale), - )) - }), - Float32 => cast_decimal_to_f32::<D, F>(array, &as_float, *scale), - Float64 => cast_decimal_to_f64::<D, F>(array, &as_float, *scale), - Utf8View => value_to_string_view(array, cast_options), - Utf8 => value_to_string::<i32>(array, cast_options), - LargeUtf8 => value_to_string::<i64>(array, cast_options), - Null => Ok(new_null_array(to_type, array.len())), - _ => Err(ArrowError::CastError(format!( - "Casting from {from_type} to {to_type} not supported" - ))), - } -} - -fn cast_to_decimal<D, M>( - array: &dyn Array, - precision: &u8, - scale: &i8, - from_type: &DataType, - to_type: &DataType, - cast_options: &CastOptions, -) -> Result<ArrayRef, ArrowError> -where - D: DecimalType + ArrowPrimitiveType<Native = M>, - M: ArrowNativeTypeOp + DecimalCast, -{ - use DataType::*; - // cast data to decimal - match from_type { - UInt8 => cast_integer_to_decimal::<_, D, M>( - array.as_primitive::<UInt8Type>(), - *precision, - *scale, - cast_options, - ), - UInt16 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<UInt16Type>(), - *precision, - *scale, - cast_options, - ), - UInt32 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<UInt32Type>(), - *precision, - *scale, - cast_options, - ), - UInt64 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<UInt64Type>(), - *precision, - *scale, - cast_options, - ), - Int8 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<Int8Type>(), - *precision, - *scale, - cast_options, - ), - Int16 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<Int16Type>(), - *precision, - *scale, - cast_options, - ), - Int32 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<Int32Type>(), - *precision, - *scale, - cast_options, - ), - Int64 => cast_integer_to_decimal::<_, D, _>( - array.as_primitive::<Int64Type>(), - *precision, - *scale, - cast_options, - ), - Float16 => cast_floating_point_to_decimal::<_, D>( - array.as_primitive::<Float16Type>(), - *precision, - *scale, - cast_options, - ), - Float32 => cast_floating_point_to_decimal::<_, D>( - array.as_primitive::<Float32Type>(), - *precision, - *scale, - cast_options, - ), - Float64 => cast_floating_point_to_decimal::<_, D>( - array.as_primitive::<Float64Type>(), - *precision, - *scale, - cast_options, - ), - Utf8View | Utf8 => { - cast_string_to_decimal::<D, i32>(array, *precision, *scale, cast_options) - } - LargeUtf8 => cast_string_to_decimal::<D, i64>(array, *precision, *scale, cast_options), - Null => Ok(new_null_array(to_type, array.len())), - _ => Err(ArrowError::CastError(format!( - "Casting from {from_type} to {to_type} not supported" - ))), - } -} - /// Number of `T` units in one day const fn units_per_day<T: ArrowTimestampType>() -> i64 { SECONDS_IN_DAY * time_unit_multiple(&T::UNIT)
