andygrove commented on code in PR #335:
URL: https://github.com/apache/datafusion-spark/pull/335#discussion_r4586995370


##########
crates/spark/src/function/datetime/time_trunc.rs:
##########
@@ -94,47 +94,171 @@ impl ScalarUDFImpl for SparkTimeTrunc {
         )))
     }
 
-    fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
-        internal_err!(
-            "spark time_trunc should have been simplified to standard 
date_trunc"
-        )
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let unit = match &args.args[0] {
+            ColumnarValue::Scalar(unit) => Some(unit),
+            ColumnarValue::Array(_) => None,
+        };
+        if !is_valid_unit(unit)? {
+            // A NULL unit returns NULL, like Spark
+            let data_type = args.return_field.data_type();
+            return 
Ok(ColumnarValue::Scalar(ScalarValue::try_from(data_type)?));
+        }
+        datafusion_functions::datetime::date_trunc().invoke_with_args(args)
     }
 
     fn simplify(
         &self,
         args: Vec<Expr>,
         info: &SimplifyContext,
     ) -> Result<ExprSimplifyResult> {
-        let fmt_expr = &args[0];
-
-        match fmt_expr.as_literal() {
-            Some(ScalarValue::Utf8(Some(unit)))
-            | Some(ScalarValue::Utf8View(Some(unit)))
-            | Some(ScalarValue::LargeUtf8(Some(unit))) => {
-                let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
-                if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
-                    return plan_err!(
-                        "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
-                    );
-                }
-            }
+        if !is_valid_unit(args[0].as_literal())? {
             // A NULL unit returns NULL, like Spark
-            Some(value) if value.is_null() => {
-                let data_type = info.get_data_type(&args[1])?;
-                return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
-                    ScalarValue::try_from(&data_type)?,
-                    None,
-                )));
-            }
-            _ => {
-                return plan_err!(
-                    "First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"
-                );
-            }
+            let data_type = info.get_data_type(&args[1])?;
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::try_from(&data_type)?,
+                None,
+            )));
         }
 
         Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
             
ScalarFunction::new_udf(datafusion_functions::datetime::date_trunc(), args),
         )))
     }
 }
+
+/// Checks the `unit` argument. Returns `false` for a NULL unit and an error 
for
+/// an invalid or non-scalar unit.
+fn is_valid_unit(unit: Option<&ScalarValue>) -> Result<bool> {
+    match unit {
+        Some(ScalarValue::Utf8(Some(unit)))
+        | Some(ScalarValue::Utf8View(Some(unit)))
+        | Some(ScalarValue::LargeUtf8(Some(unit))) => {
+            let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
+            if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
+                return plan_err!(
+                    "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
+                );
+            }
+            Ok(true)
+        }
+        Some(value) if value.is_null() => Ok(false),
+        _ => plan_err!("First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"),

Review Comment:
   nit (pre-existing): "must be non-null scalar Utf8" is a bit misleading now 
that a NULL unit is explicitly accepted and returns NULL. Maybe "must be a 
constant string"?



##########
crates/spark/src/function/datetime/time_trunc.rs:
##########
@@ -94,47 +94,171 @@ impl ScalarUDFImpl for SparkTimeTrunc {
         )))
     }
 
-    fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
-        internal_err!(
-            "spark time_trunc should have been simplified to standard 
date_trunc"
-        )
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let unit = match &args.args[0] {
+            ColumnarValue::Scalar(unit) => Some(unit),
+            ColumnarValue::Array(_) => None,
+        };
+        if !is_valid_unit(unit)? {
+            // A NULL unit returns NULL, like Spark
+            let data_type = args.return_field.data_type();
+            return 
Ok(ColumnarValue::Scalar(ScalarValue::try_from(data_type)?));
+        }
+        datafusion_functions::datetime::date_trunc().invoke_with_args(args)
     }
 
     fn simplify(
         &self,
         args: Vec<Expr>,
         info: &SimplifyContext,
     ) -> Result<ExprSimplifyResult> {
-        let fmt_expr = &args[0];
-
-        match fmt_expr.as_literal() {
-            Some(ScalarValue::Utf8(Some(unit)))
-            | Some(ScalarValue::Utf8View(Some(unit)))
-            | Some(ScalarValue::LargeUtf8(Some(unit))) => {
-                let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
-                if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
-                    return plan_err!(
-                        "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
-                    );
-                }
-            }
+        if !is_valid_unit(args[0].as_literal())? {
             // A NULL unit returns NULL, like Spark
-            Some(value) if value.is_null() => {
-                let data_type = info.get_data_type(&args[1])?;
-                return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
-                    ScalarValue::try_from(&data_type)?,
-                    None,
-                )));
-            }
-            _ => {
-                return plan_err!(
-                    "First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"
-                );
-            }
+            let data_type = info.get_data_type(&args[1])?;
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::try_from(&data_type)?,
+                None,
+            )));
         }
 
         Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
             
ScalarFunction::new_udf(datafusion_functions::datetime::date_trunc(), args),
         )))
     }
 }
+
+/// Checks the `unit` argument. Returns `false` for a NULL unit and an error 
for
+/// an invalid or non-scalar unit.
+fn is_valid_unit(unit: Option<&ScalarValue>) -> Result<bool> {
+    match unit {
+        Some(ScalarValue::Utf8(Some(unit)))
+        | Some(ScalarValue::Utf8View(Some(unit)))
+        | Some(ScalarValue::LargeUtf8(Some(unit))) => {
+            let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
+            if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
+                return plan_err!(
+                    "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
+                );
+            }
+            Ok(true)
+        }
+        Some(value) if value.is_null() => Ok(false),
+        _ => plan_err!("First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use arrow::array::{Array, ArrayRef, Time64NanosecondArray};
+    use datafusion_common::config::ConfigOptions;
+
+    /// `HH:MM:SS.ffffff` as nanoseconds since midnight
+    fn nanos(hours: i64, minutes: i64, seconds: i64, micros: i64) -> i64 {
+        ((hours * 60 + minutes) * 60 + seconds) * 1_000_000_000 + micros * 
1_000
+    }
+
+    /// Calls `invoke_with_args` directly, without `simplify`
+    fn invoke(unit: ScalarValue, time: ColumnarValue) -> Result<ColumnarValue> 
{
+        let time_type = time.data_type();
+        let number_rows = match &time {
+            ColumnarValue::Array(array) => array.len(),
+            ColumnarValue::Scalar(_) => 1,
+        };
+        SparkTimeTrunc::new().invoke_with_args(ScalarFunctionArgs {
+            arg_fields: vec![
+                Arc::new(Field::new("unit", unit.data_type(), true)),
+                Arc::new(Field::new("time", time_type.clone(), true)),
+            ],
+            args: vec![ColumnarValue::Scalar(unit), time],
+            number_rows,
+            return_field: Arc::new(Field::new("time_trunc", time_type, true)),
+            config_options: Arc::new(ConfigOptions::default()),
+        })
+    }
+
+    fn scalar_time(value: i64) -> ColumnarValue {

Review Comment:
   nit: the unit tests only use `Time64(Nanosecond)`. A `Time32(Second)` or 
`Time32(Millisecond)` case would cover the other branch of the `Time` signature 
class. It works when I run it end to end, so this is just about coverage.



##########
crates/spark/src/function/datetime/time_trunc.rs:
##########
@@ -94,47 +94,171 @@ impl ScalarUDFImpl for SparkTimeTrunc {
         )))
     }
 
-    fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
-        internal_err!(
-            "spark time_trunc should have been simplified to standard 
date_trunc"
-        )
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let unit = match &args.args[0] {
+            ColumnarValue::Scalar(unit) => Some(unit),
+            ColumnarValue::Array(_) => None,
+        };
+        if !is_valid_unit(unit)? {
+            // A NULL unit returns NULL, like Spark
+            let data_type = args.return_field.data_type();
+            return 
Ok(ColumnarValue::Scalar(ScalarValue::try_from(data_type)?));
+        }
+        datafusion_functions::datetime::date_trunc().invoke_with_args(args)
     }
 
     fn simplify(
         &self,
         args: Vec<Expr>,
         info: &SimplifyContext,
     ) -> Result<ExprSimplifyResult> {
-        let fmt_expr = &args[0];
-
-        match fmt_expr.as_literal() {
-            Some(ScalarValue::Utf8(Some(unit)))
-            | Some(ScalarValue::Utf8View(Some(unit)))
-            | Some(ScalarValue::LargeUtf8(Some(unit))) => {
-                let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
-                if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
-                    return plan_err!(
-                        "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
-                    );
-                }
-            }
+        if !is_valid_unit(args[0].as_literal())? {
             // A NULL unit returns NULL, like Spark
-            Some(value) if value.is_null() => {
-                let data_type = info.get_data_type(&args[1])?;
-                return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
-                    ScalarValue::try_from(&data_type)?,
-                    None,
-                )));
-            }
-            _ => {
-                return plan_err!(
-                    "First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"
-                );
-            }
+            let data_type = info.get_data_type(&args[1])?;
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::try_from(&data_type)?,
+                None,
+            )));
         }
 
         Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
             
ScalarFunction::new_udf(datafusion_functions::datetime::date_trunc(), args),
         )))
     }
 }
+
+/// Checks the `unit` argument. Returns `false` for a NULL unit and an error 
for
+/// an invalid or non-scalar unit.
+fn is_valid_unit(unit: Option<&ScalarValue>) -> Result<bool> {
+    match unit {
+        Some(ScalarValue::Utf8(Some(unit)))
+        | Some(ScalarValue::Utf8View(Some(unit)))
+        | Some(ScalarValue::LargeUtf8(Some(unit))) => {
+            let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
+            if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
+                return plan_err!(

Review Comment:
   nit: on the `invoke_with_args` path this now surfaces as `Error during 
planning: ...` at execution time. `exec_err!` would be more accurate there, 
e.g. by having the caller pick the error kind. The message text is the same 
either way, so this is cosmetic.



##########
crates/spark/src/function/datetime/time_trunc.rs:
##########
@@ -94,47 +94,171 @@ impl ScalarUDFImpl for SparkTimeTrunc {
         )))
     }
 
-    fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
-        internal_err!(
-            "spark time_trunc should have been simplified to standard 
date_trunc"
-        )
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let unit = match &args.args[0] {
+            ColumnarValue::Scalar(unit) => Some(unit),
+            ColumnarValue::Array(_) => None,
+        };
+        if !is_valid_unit(unit)? {
+            // A NULL unit returns NULL, like Spark
+            let data_type = args.return_field.data_type();
+            return 
Ok(ColumnarValue::Scalar(ScalarValue::try_from(data_type)?));
+        }
+        datafusion_functions::datetime::date_trunc().invoke_with_args(args)
     }
 
     fn simplify(
         &self,
         args: Vec<Expr>,
         info: &SimplifyContext,
     ) -> Result<ExprSimplifyResult> {
-        let fmt_expr = &args[0];
-
-        match fmt_expr.as_literal() {
-            Some(ScalarValue::Utf8(Some(unit)))
-            | Some(ScalarValue::Utf8View(Some(unit)))
-            | Some(ScalarValue::LargeUtf8(Some(unit))) => {
-                let valid = ["HOUR", "MINUTE", "SECOND", "MILLISECOND", 
"MICROSECOND"];
-                if !valid.iter().any(|v| v.eq_ignore_ascii_case(unit)) {
-                    return plan_err!(
-                        "The value of parameter(s) `unit` in `time_trunc` is 
invalid: expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND', 
'MICROSECOND', but got '{unit}'. SQLSTATE: 22023"
-                    );
-                }
-            }
+        if !is_valid_unit(args[0].as_literal())? {
             // A NULL unit returns NULL, like Spark
-            Some(value) if value.is_null() => {
-                let data_type = info.get_data_type(&args[1])?;
-                return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
-                    ScalarValue::try_from(&data_type)?,
-                    None,
-                )));
-            }
-            _ => {
-                return plan_err!(
-                    "First argument of `TIME_TRUNC` must be non-null scalar 
Utf8"
-                );
-            }
+            let data_type = info.get_data_type(&args[1])?;
+            return Ok(ExprSimplifyResult::Simplified(Expr::Literal(
+                ScalarValue::try_from(&data_type)?,
+                None,
+            )));
         }
 
         Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
             
ScalarFunction::new_udf(datafusion_functions::datetime::date_trunc(), args),
         )))
     }
 }
+
+/// Checks the `unit` argument. Returns `false` for a NULL unit and an error 
for
+/// an invalid or non-scalar unit.
+fn is_valid_unit(unit: Option<&ScalarValue>) -> Result<bool> {

Review Comment:
   nit: `is_valid_unit` returns `Ok(false)` for a NULL unit, so 
`!is_valid_unit(..)` at the call sites reads as "invalid unit" when it actually 
means "NULL unit". A name like `check_unit`, or returning something like 
`Result<Option<()>>` or a small enum, would make the NULL case clearer. Not 
blocking.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to