This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 9095c867a chore: Refactor bit_not (#2896)
9095c867a is described below
commit 9095c867a35133816dc0802c6d58843968962a0a
Author: Kazantsev Maksim <[email protected]>
AuthorDate: Sat Dec 13 08:34:44 2025 -0800
chore: Refactor bit_not (#2896)
Co-authored-by: Kazantsev Maksim <[email protected]>
---
native/core/src/execution/jni_api.rs | 2 +
native/spark-expr/src/bitwise_funcs/bitwise_not.rs | 150 ---------------------
native/spark-expr/src/bitwise_funcs/mod.rs | 2 -
native/spark-expr/src/comet_scalar_funcs.rs | 5 +-
.../scala/org/apache/comet/serde/bitwise.scala | 2 +-
5 files changed, 5 insertions(+), 156 deletions(-)
diff --git a/native/core/src/execution/jni_api.rs
b/native/core/src/execution/jni_api.rs
index b6c933d0b..a24d99305 100644
--- a/native/core/src/execution/jni_api.rs
+++ b/native/core/src/execution/jni_api.rs
@@ -41,6 +41,7 @@ use datafusion::{
};
use datafusion_comet_proto::spark_operator::Operator;
use datafusion_spark::function::bitwise::bit_get::SparkBitGet;
+use datafusion_spark::function::bitwise::bitwise_not::SparkBitwiseNot;
use datafusion_spark::function::datetime::date_add::SparkDateAdd;
use datafusion_spark::function::datetime::date_sub::SparkDateSub;
use datafusion_spark::function::hash::sha1::SparkSha1;
@@ -335,6 +336,7 @@ fn register_datafusion_spark_function(session_ctx:
&SessionContext) {
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkDateSub::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha1::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkConcat::default()));
+
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBitwiseNot::default()));
}
/// Prepares arrow arrays for output.
diff --git a/native/spark-expr/src/bitwise_funcs/bitwise_not.rs
b/native/spark-expr/src/bitwise_funcs/bitwise_not.rs
deleted file mode 100644
index 45dd0b312..000000000
--- a/native/spark-expr/src/bitwise_funcs/bitwise_not.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use arrow::{array::*, datatypes::DataType};
-use datafusion::common::{
- exec_err, internal_datafusion_err, internal_err, DataFusionError, Result,
-};
-use datafusion::logical_expr::{ColumnarValue, Volatility};
-use datafusion::logical_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};
-use std::{any::Any, sync::Arc};
-
-#[derive(Debug, PartialEq, Eq, Hash)]
-pub struct SparkBitwiseNot {
- signature: Signature,
- aliases: Vec<String>,
-}
-
-impl Default for SparkBitwiseNot {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl SparkBitwiseNot {
- pub fn new() -> Self {
- Self {
- signature: Signature::user_defined(Volatility::Immutable),
- aliases: vec![],
- }
- }
-}
-
-impl ScalarUDFImpl for SparkBitwiseNot {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn name(&self) -> &str {
- "bit_not"
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
- Ok(match arg_types[0] {
- DataType::Int8 => DataType::Int8,
- DataType::Int16 => DataType::Int16,
- DataType::Int32 => DataType::Int32,
- DataType::Int64 => DataType::Int64,
- DataType::Null => DataType::Null,
- _ => return exec_err!("{} function can only accept integral
arrays", self.name()),
- })
- }
-
- fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
- let args: [ColumnarValue; 1] = args
- .args
- .try_into()
- .map_err(|_| internal_datafusion_err!("bit_not expects exactly one
argument"))?;
- bitwise_not(args)
- }
-
- fn aliases(&self) -> &[String] {
- &self.aliases
- }
-}
-
-macro_rules! compute_op {
- ($OPERAND:expr, $DT:ident) => {{
- let operand = $OPERAND.as_any().downcast_ref::<$DT>().ok_or_else(|| {
- DataFusionError::Execution(format!(
- "compute_op failed to downcast array to: {:?}",
- stringify!($DT)
- ))
- })?;
- let result: $DT = operand.iter().map(|x| x.map(|y| !y)).collect();
- Ok(Arc::new(result))
- }};
-}
-
-pub fn bitwise_not(args: [ColumnarValue; 1]) -> Result<ColumnarValue> {
- match args {
- [ColumnarValue::Array(array)] => {
- let result: Result<ArrayRef> = match array.data_type() {
- DataType::Int8 => compute_op!(array, Int8Array),
- DataType::Int16 => compute_op!(array, Int16Array),
- DataType::Int32 => compute_op!(array, Int32Array),
- DataType::Int64 => compute_op!(array, Int64Array),
- _ => exec_err!("bit_not can't be evaluated because the
expression's type is {:?}, not signed int", array.data_type()),
- };
- result.map(ColumnarValue::Array)
- }
- [ColumnarValue::Scalar(_)] => internal_err!("shouldn't go to bitwise
not scalar path"),
- }
-}
-
-#[cfg(test)]
-mod tests {
- use datafusion::common::{cast::as_int32_array, Result};
-
- use super::*;
-
- #[test]
- fn bitwise_not_op() -> Result<()> {
- let int_array = Int32Array::from(vec![
- Some(1),
- Some(2),
- None,
- Some(12345),
- Some(89),
- Some(-3456),
- ]);
- let expected = &Int32Array::from(vec![
- Some(-2),
- Some(-3),
- None,
- Some(-12346),
- Some(-90),
- Some(3455),
- ]);
-
- let columnar_value = ColumnarValue::Array(Arc::new(int_array));
-
- let result = bitwise_not([columnar_value])?;
- let result = match result {
- ColumnarValue::Array(array) => array,
- _ => panic!("Expected array"),
- };
- let result = as_int32_array(&result).expect("failed to downcast to
In32Array");
- assert_eq!(result, expected);
-
- Ok(())
- }
-}
diff --git a/native/spark-expr/src/bitwise_funcs/mod.rs
b/native/spark-expr/src/bitwise_funcs/mod.rs
index 3f148a6dc..d96857e23 100644
--- a/native/spark-expr/src/bitwise_funcs/mod.rs
+++ b/native/spark-expr/src/bitwise_funcs/mod.rs
@@ -16,7 +16,5 @@
// under the License.
mod bitwise_count;
-mod bitwise_not;
pub use bitwise_count::SparkBitwiseCount;
-pub use bitwise_not::SparkBitwiseNot;
diff --git a/native/spark-expr/src/comet_scalar_funcs.rs
b/native/spark-expr/src/comet_scalar_funcs.rs
index 021bb1c78..e9f1caa8a 100644
--- a/native/spark-expr/src/comet_scalar_funcs.rs
+++ b/native/spark-expr/src/comet_scalar_funcs.rs
@@ -22,8 +22,8 @@ use crate::math_funcs::modulo_expr::spark_modulo;
use crate::{
spark_array_repeat, spark_ceil, spark_decimal_div,
spark_decimal_integral_div, spark_floor,
spark_hex, spark_isnan, spark_lpad, spark_make_decimal,
spark_read_side_padding, spark_round,
- spark_rpad, spark_unhex, spark_unscaled_value, EvalMode,
SparkBitwiseCount, SparkBitwiseNot,
- SparkDateTrunc, SparkStringSpace,
+ spark_rpad, spark_unhex, spark_unscaled_value, EvalMode,
SparkBitwiseCount, SparkDateTrunc,
+ SparkStringSpace,
};
use arrow::datatypes::DataType;
use datafusion::common::{DataFusionError, Result as DataFusionResult};
@@ -195,7 +195,6 @@ pub fn create_comet_physical_fun_with_eval_mode(
fn all_scalar_functions() -> Vec<Arc<ScalarUDF>> {
vec![
- Arc::new(ScalarUDF::new_from_impl(SparkBitwiseNot::default())),
Arc::new(ScalarUDF::new_from_impl(SparkBitwiseCount::default())),
Arc::new(ScalarUDF::new_from_impl(SparkDateTrunc::default())),
Arc::new(ScalarUDF::new_from_impl(SparkStringSpace::default())),
diff --git a/spark/src/main/scala/org/apache/comet/serde/bitwise.scala
b/spark/src/main/scala/org/apache/comet/serde/bitwise.scala
index 56404fe3c..8020502ab 100644
--- a/spark/src/main/scala/org/apache/comet/serde/bitwise.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/bitwise.scala
@@ -46,7 +46,7 @@ object CometBitwiseNot extends
CometExpressionSerde[BitwiseNot] {
binding: Boolean): Option[ExprOuterClass.Expr] = {
val childProto = exprToProto(expr.child, inputs, binding)
val bitNotScalarExpr =
- scalarFunctionExprToProto("bit_not", childProto)
+ scalarFunctionExprToProto("bitwise_not", childProto)
optExprWithInfo(bitNotScalarExpr, expr, expr.children: _*)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
