comphead commented on code in PR #7661: URL: https://github.com/apache/datafusion-comet/pull/7661#discussion_r4608073722
########## native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs: ########## @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::RecordBatch; +use datafusion::common::{DataFusionError, Result}; +use datafusion::logical_expr::ColumnarValue; +use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_jni_bridge::errors::CometError; + +use crate::{SparkError, SparkErrorWithContext}; + +/// Spark's `CheckOverflowInTableInsert`: evaluates the cast that a table insert added and turns +/// any `SparkArithmeticException` from it into `CAST_OVERFLOW_IN_TABLE_INSERT`, which names the +/// column instead of the value and carries no query context. +#[derive(Debug)] +pub struct CheckOverflowInTableInsert { + child: Arc<dyn PhysicalExpr>, + from_type: String, + to_type: String, + column_name: String, +} + +impl CheckOverflowInTableInsert { + pub fn new( + child: Arc<dyn PhysicalExpr>, + from_type: String, + to_type: String, + column_name: String, + ) -> Self { + Self { + child, + from_type, + to_type, + column_name, + } + } + + fn table_insert_error(&self) -> DataFusionError { + SparkError::CastOverflowInTableInsert { + from_type: self.from_type.clone(), + to_type: self.to_type.clone(), + column_name: self.column_name.clone(), + } + .into() + } +} + +impl PartialEq for CheckOverflowInTableInsert { + fn eq(&self, other: &Self) -> bool { + self.child.eq(&other.child) + && self.from_type == other.from_type + && self.to_type == other.to_type + && self.column_name == other.column_name + } +} + +impl Eq for CheckOverflowInTableInsert {} + +impl Hash for CheckOverflowInTableInsert { + fn hash<H: Hasher>(&self, state: &mut H) { + self.child.hash(state); + self.from_type.hash(state); + self.to_type.hash(state); + self.column_name.hash(state); + } +} + +impl Display for CheckOverflowInTableInsert { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CheckOverflowInTableInsert({}, column: {})", + self.child, self.column_name + ) + } +} + +impl PhysicalExpr for CheckOverflowInTableInsert { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.child.fmt_sql(f) + } + + fn data_type(&self, input_schema: &Schema) -> Result<DataType> { + self.child.data_type(input_schema) + } + + fn nullable(&self, input_schema: &Schema) -> Result<bool> { + self.child.nullable(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { + match self.child.evaluate(batch) { + Err(e) if is_arithmetic_error(&e) => Err(self.table_insert_error()), + other => other, + } + } + + fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> { + vec![&self.child] + } + + fn with_new_children( + self: Arc<Self>, + children: Vec<Arc<dyn PhysicalExpr>>, + ) -> Result<Arc<dyn PhysicalExpr>> { + match children.as_slice() { + [child] => Ok(Arc::new(Self::new( + Arc::clone(child), + self.from_type.clone(), + self.to_type.clone(), + self.column_name.clone(), + ))), + _ => Err(DataFusionError::Internal( + "CheckOverflowInTableInsert should have exactly one child".to_string(), + )), + } + } +} + +/// Whether Spark raises this error as a `SparkArithmeticException`: a native `SparkError`, or an +/// exception thrown by a child that runs Spark's own code through the JVM codegen dispatcher. +/// Only the top of a `DataFusionError::External` is checked, because that is where the JNI layer +/// reads a Spark error from. +fn is_arithmetic_error(error: &DataFusionError) -> bool { + let DataFusionError::External(e) = error else { + return false; + }; + if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::<CometError>() { + return class == "org.apache.spark.SparkArithmeticException"; + } + e.downcast_ref::<SparkError>() + .or_else(|| e.downcast_ref::<SparkErrorWithContext>().map(|e| &e.error)) + .is_some_and(SparkError::is_arithmetic) +} Review Comment: #4653 landed on `main` after this branch was cut. `jvm_exception` in `native/jni-bridge/src/errors.rs` now finds the Spark payload anywhere in the `source()` chain (`extract_spark_payload`, built on `datafusion_comet_common::error_chain`). So the claim in this doc comment that the JNI layer only reads the top `DataFusionError::External` is no longer true. This function still only inspects the top `External`. A child error that reaches here wrapped, for example in `Context`, `Shared` or `ArrowError::ExternalError`, is not recognised. I expect it to keep its own class, such as `DIVIDE_BY_ZERO`, where Spark reports `CAST_OVERFLOW_IN_TABLE_INSERT`. That is the dispatched ScalaUDF case #4653 fixes. I did not run this. Suggest walking the chain the same way, taking the first Spark-typed payload in the order `extract_spark_payload` uses (`CometError::JavaException`, then `SparkErrorWithContext`, then `SparkError`): ```rust fn is_arithmetic_error(error: &DataFusionError) -> bool { error_chain(error) .find_map(|e| { if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::<CometError>() { Some(class == "org.apache.spark.SparkArithmeticException") } else if let Some(ctx) = e.downcast_ref::<SparkErrorWithContext>() { Some(ctx.error.is_arithmetic()) } else { e.downcast_ref::<SparkError>().map(SparkError::is_arithmetic) } }) .unwrap_or(false) } ``` `error_chain` is exported from `datafusion_comet_common` and `checked_binary_expr.rs` already imports it. A unit test with the child error wrapped in `DataFusionError::Context` would cover it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
