This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 2d56253893 fix(spark): Use wrapping addition/subtraction in
`SparkDateAdd` and `SparkDateSub` (#19377)
2d56253893 is described below
commit 2d562538930eaf8a25e9e77da4b46281992a947c
Author: Mikhail Zabaluev <[email protected]>
AuthorDate: Tue Jan 6 03:14:01 2026 +0200
fix(spark): Use wrapping addition/subtraction in `SparkDateAdd` and
`SparkDateSub` (#19377)
## Which issue does this PR close?
- Addresses
[#17987(comment)](https://github.com/apache/datafusion/issues/17987#issuecomment-3666497393).
## Rationale for this change
In Spark, `date_add` and `date_sub` functions do not raise an error when
the addition or subtraction overflows 32-bit integer, regardless of
whether the ANSI evaluation mode is in effect.
## What changes are included in this PR?
Changed the implementations to use `wrapping_add` and `wrapping_sub`.
## Are these changes tested?
Updated `date_add.slt` to expect overflowed results rather than the
errors.
## Are there any user-facing changes?
The functions behave more like Spark, even if Spark is doing the wrong
thing.
---
datafusion/spark/src/function/datetime/date_add.rs | 25 ++++++----------------
datafusion/spark/src/function/datetime/date_sub.rs | 25 ++++++----------------
.../test_files/spark/datetime/date_add.slt | 12 +++++++----
3 files changed, 20 insertions(+), 42 deletions(-)
diff --git a/datafusion/spark/src/function/datetime/date_add.rs
b/datafusion/spark/src/function/datetime/date_add.rs
index b176f51ae6..78b9c904ce 100644
--- a/datafusion/spark/src/function/datetime/date_add.rs
+++ b/datafusion/spark/src/function/datetime/date_add.rs
@@ -21,7 +21,6 @@ use std::sync::Arc;
use arrow::array::ArrayRef;
use arrow::compute;
use arrow::datatypes::{DataType, Date32Type, Field, FieldRef};
-use arrow::error::ArrowError;
use datafusion_common::cast::{
as_date32_array, as_int8_array, as_int16_array, as_int32_array,
};
@@ -107,38 +106,26 @@ fn spark_date_add(args: &[ArrayRef]) -> Result<ArrayRef> {
let result = match days_arg.data_type() {
DataType::Int8 => {
let days_array = as_int8_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_add(days as i32).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_add".to_string())
- })
- },
+ |date, days| date.wrapping_add(days as i32),
)?
}
DataType::Int16 => {
let days_array = as_int16_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_add(days as i32).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_add".to_string())
- })
- },
+ |date, days| date.wrapping_add(days as i32),
)?
}
DataType::Int32 => {
let days_array = as_int32_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_add(days).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_add".to_string())
- })
- },
+ |date, days| date.wrapping_add(days),
)?
}
_ => {
diff --git a/datafusion/spark/src/function/datetime/date_sub.rs
b/datafusion/spark/src/function/datetime/date_sub.rs
index 7e56670f17..34894317f6 100644
--- a/datafusion/spark/src/function/datetime/date_sub.rs
+++ b/datafusion/spark/src/function/datetime/date_sub.rs
@@ -21,7 +21,6 @@ use std::sync::Arc;
use arrow::array::ArrayRef;
use arrow::compute;
use arrow::datatypes::{DataType, Date32Type, Field, FieldRef};
-use arrow::error::ArrowError;
use datafusion_common::cast::{
as_date32_array, as_int8_array, as_int16_array, as_int32_array,
};
@@ -105,38 +104,26 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result<ArrayRef> {
let result = match days_arg.data_type() {
DataType::Int8 => {
let days_array = as_int8_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_sub(days as i32).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_sub".to_string())
- })
- },
+ |date, days| date.wrapping_sub(days as i32),
)?
}
DataType::Int16 => {
let days_array = as_int16_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_sub(days as i32).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_sub".to_string())
- })
- },
+ |date, days| date.wrapping_sub(days as i32),
)?
}
DataType::Int32 => {
let days_array = as_int32_array(days_arg)?;
- compute::try_binary::<_, _, _, Date32Type>(
+ compute::binary::<_, _, _, Date32Type>(
date_array,
days_array,
- |date, days| {
- date.checked_sub(days).ok_or_else(|| {
- ArrowError::ArithmeticOverflow("date_sub".to_string())
- })
- },
+ |date, days| date.wrapping_sub(days),
)?
}
_ => {
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt
b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt
index 2e9851ca1e..a2ac7cf2ed 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt
@@ -45,11 +45,15 @@ SELECT date_sub('2016-07-30'::date, 0::int);
----
2016-07-30
-query error DataFusion error: Arrow error: Arithmetic overflow: date_add
-SELECT date_add('2016-07-30'::date, 2147483647::int);
+query I
+SELECT date_add('2016-07-30'::date, 2147483647::int)::int;
+----
+-2147466637
-query error DataFusion error: Arrow error: Arithmetic overflow: date_sub
-SELECT date_sub('1969-01-01'::date, 2147483647::int);
+query I
+SELECT date_sub('1969-01-01'::date, 2147483647::int)::int;
+----
+2147483284
query D
SELECT date_add('2016-07-30'::date, 100000::int);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
