This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new a81ab3af8f Decimal type support for `to_timestamp` (#15486)
a81ab3af8f is described below
commit a81ab3af8f9f565735120ef75ab7f64d8e53f88e
Author: Jagdish Parihar <[email protected]>
AuthorDate: Mon Mar 31 11:06:34 2025 -0700
Decimal type support for `to_timestamp` (#15486)
* wip: decimal type support for to_timestamp
* updated timestamp handling for cast operator
* fixed clippy error
* fixed fmt
* updated timestamps.slt file for decimal128 type support
---
datafusion/common/src/scalar/mod.rs | 28 ++++++++++++++++++++
datafusion/functions/src/datetime/to_timestamp.rs | 31 ++++++++++++++++++++---
datafusion/sqllogictest/test_files/timestamps.slt | 27 ++++++++++++++++++++
3 files changed, 82 insertions(+), 4 deletions(-)
diff --git a/datafusion/common/src/scalar/mod.rs
b/datafusion/common/src/scalar/mod.rs
index 2b758f4568..4a58530edf 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -3036,6 +3036,34 @@ impl ScalarValue {
DataType::Timestamp(TimeUnit::Nanosecond, None),
) => ScalarValue::Int64(Some((float_ts *
1_000_000_000_f64).trunc() as i64))
.to_array()?,
+ (
+ ScalarValue::Decimal128(Some(decimal_value), _, scale),
+ DataType::Timestamp(time_unit, None),
+ ) => {
+ let scale_factor = 10_i128.pow(*scale as u32);
+ let seconds = decimal_value / scale_factor;
+ let fraction = decimal_value % scale_factor;
+
+ let timestamp_value = match time_unit {
+ TimeUnit::Second => ScalarValue::Int64(Some(seconds as
i64)),
+ TimeUnit::Millisecond => {
+ let millis = seconds * 1_000 + (fraction * 1_000) /
scale_factor;
+ ScalarValue::Int64(Some(millis as i64))
+ }
+ TimeUnit::Microsecond => {
+ let micros =
+ seconds * 1_000_000 + (fraction * 1_000_000) /
scale_factor;
+ ScalarValue::Int64(Some(micros as i64))
+ }
+ TimeUnit::Nanosecond => {
+ let nanos = seconds * 1_000_000_000
+ + (fraction * 1_000_000_000) / scale_factor;
+ ScalarValue::Int64(Some(nanos as i64))
+ }
+ };
+
+ timestamp_value.to_array()?
+ }
_ => self.to_array()?,
};
diff --git a/datafusion/functions/src/datetime/to_timestamp.rs
b/datafusion/functions/src/datetime/to_timestamp.rs
index f1c61fe2b9..52c86733f3 100644
--- a/datafusion/functions/src/datetime/to_timestamp.rs
+++ b/datafusion/functions/src/datetime/to_timestamp.rs
@@ -18,15 +18,14 @@
use std::any::Any;
use std::sync::Arc;
+use crate::datetime::common::*;
use arrow::datatypes::DataType::*;
use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second};
use arrow::datatypes::{
ArrowTimestampType, DataType, TimeUnit, TimestampMicrosecondType,
TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
};
-
-use crate::datetime::common::*;
-use datafusion_common::{exec_err, Result, ScalarType};
+use datafusion_common::{exec_err, Result, ScalarType, ScalarValue};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,
};
@@ -329,6 +328,30 @@ impl ScalarUDFImpl for ToTimestampFunc {
Utf8View | LargeUtf8 | Utf8 => {
to_timestamp_impl::<TimestampNanosecondType>(&args,
"to_timestamp")
}
+ Decimal128(_, _) => {
+ match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Decimal128(
+ Some(value),
+ _,
+ scale,
+ )) => {
+ // Convert decimal to seconds and nanoseconds
+ let scale_factor = 10_i128.pow(*scale as u32);
+ let seconds = value / scale_factor;
+ let fraction = value % scale_factor;
+
+ let nanos = (fraction * 1_000_000_000) / scale_factor;
+
+ let timestamp_nanos = seconds * 1_000_000_000 + nanos;
+
+
Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ Some(timestamp_nanos as i64),
+ None,
+ )))
+ }
+ _ => exec_err!("Invalid decimal value"),
+ }
+ }
other => {
exec_err!(
"Unsupported data type {:?} for function to_timestamp",
@@ -377,7 +400,7 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc {
}
match args[0].data_type() {
- Null | Int32 | Int64 | Timestamp(_, None) => {
+ Null | Int32 | Int64 | Timestamp(_, None) | Decimal128(_, _) => {
args[0].cast_to(&Timestamp(Second, None), None)
}
Timestamp(_, Some(tz)) => args[0].cast_to(&Timestamp(Second,
Some(tz)), None),
diff --git a/datafusion/sqllogictest/test_files/timestamps.slt
b/datafusion/sqllogictest/test_files/timestamps.slt
index dcbcfbfa43..e042e3863b 100644
--- a/datafusion/sqllogictest/test_files/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/timestamps.slt
@@ -416,6 +416,33 @@ SELECT to_timestamp(123456789.123456789) as c1,
cast(123456789.123456789 as time
----
1973-11-29T21:33:09.123456784 1973-11-29T21:33:09.123456784
1973-11-29T21:33:09.123456784
+# to_timestamp Decimal128 inputs
+
+query PPP
+SELECT to_timestamp(arrow_cast(1.1, 'Decimal128(2,1)')) as c1,
cast(arrow_cast(1.1, 'Decimal128(2,1)') as timestamp) as c2, arrow_cast(1.1,
'Decimal128(2,1)')::timestamp as c3;
+----
+1970-01-01T00:00:01.100 1970-01-01T00:00:01.100 1970-01-01T00:00:01.100
+
+query PPP
+SELECT to_timestamp(arrow_cast(-1.1, 'Decimal128(2,1)')) as c1,
cast(arrow_cast(-1.1, 'Decimal128(2,1)') as timestamp) as c2, arrow_cast(-1.1,
'Decimal128(2,1)')::timestamp as c3;
+----
+1969-12-31T23:59:58.900 1969-12-31T23:59:58.900 1969-12-31T23:59:58.900
+
+query PPP
+SELECT to_timestamp(arrow_cast(0.0, 'Decimal128(2,1)')) as c1,
cast(arrow_cast(0.0, 'Decimal128(2,1)') as timestamp) as c2, arrow_cast(0.0,
'Decimal128(2,1)')::timestamp as c3;
+----
+1970-01-01T00:00:00 1970-01-01T00:00:00 1970-01-01T00:00:00
+
+query PPP
+SELECT to_timestamp(arrow_cast(1.23456789, 'Decimal128(9,8)')) as c1,
cast(arrow_cast(1.23456789, 'Decimal128(9,8)') as timestamp) as c2,
arrow_cast(1.23456789, 'Decimal128(9,8)')::timestamp as c3;
+----
+1970-01-01T00:00:01.234567890 1970-01-01T00:00:01.234567890
1970-01-01T00:00:01.234567890
+
+query PPP
+SELECT to_timestamp(arrow_cast(123456789.123456789, 'Decimal128(18,9)')) as
c1, cast(arrow_cast(123456789.123456789, 'Decimal128(18,9)') as timestamp) as
c2, arrow_cast(123456789.123456789, 'Decimal128(18,9)')::timestamp as c3;
+----
+1973-11-29T21:33:09.123456784 1973-11-29T21:33:09.123456784
1973-11-29T21:33:09.123456784
+
# from_unixtime
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
