This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 042f27672 chore: Fallback to Spark for `array_reverse` for
`array<binary>` (#2759)
042f27672 is described below
commit 042f276722ff0415d1be0e3ad0dd163827e29480
Author: Oleks V <[email protected]>
AuthorDate: Tue Nov 11 11:45:02 2025 -0800
chore: Fallback to Spark for `array_reverse` for `array<binary>` (#2759)
* chore: Fallback to Spark for `array_reverse` for binary
---
.../main/scala/org/apache/comet/serde/arrays.scala | 21 +++++++++++++++-
.../apache/comet/CometArrayExpressionSuite.scala | 29 ++++++++++++++++++++++
2 files changed, 49 insertions(+), 1 deletion(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
index e8a4ce76f..f59234d40 100644
--- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
@@ -435,6 +435,25 @@ object CometGetArrayItem extends
CometExpressionSerde[GetArrayItem] {
}
object CometArrayReverse extends CometExpressionSerde[Reverse] with ArraysBase
{
+ val unsupportedReason = "reverse on array containing binary is not supported"
+
+ @tailrec
+ private def containsBinary(dt: DataType): Boolean = {
+ dt match {
+ case BinaryType => true
+ case ArrayType(elementType, _) => containsBinary(elementType)
+ case _ => false
+ }
+ }
+
+ override def getSupportLevel(expr: Reverse): SupportLevel = {
+ if (containsBinary(expr.child.dataType)) {
+ Incompatible(Some(unsupportedReason))
+ } else {
+ Compatible(None)
+ }
+ }
+
override def convert(
expr: Reverse,
inputs: Seq[Attribute],
@@ -474,7 +493,7 @@ object CometElementAt extends
CometExpressionSerde[ElementAt] {
.setOneBased(true)
.setFailOnError(expr.failOnError)
- defaultExpr.foreach(arrayExtractBuilder.setDefaultValue(_))
+ defaultExpr.foreach(arrayExtractBuilder.setDefaultValue)
Some(
ExprOuterClass.Expr
diff --git
a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
index 758311f1f..a8f5c66dd 100644
--- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala
@@ -773,4 +773,33 @@ class CometArrayExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelp
}
}
}
+
+ // https://github.com/apache/datafusion-comet/issues/2612
+ test("array_reverse - fallback for binary array") {
+ val fallbackReason =
+ if (CometConf.COMET_NATIVE_SCAN_IMPL.key == CometConf.SCAN_NATIVE_COMET
|| sys.env
+ .getOrElse("COMET_PARQUET_SCAN_IMPL", "") ==
CometConf.SCAN_NATIVE_COMET) {
+ "Unsupported schema"
+ } else {
+ CometArrayReverse.unsupportedReason
+ }
+ withTable("t1") {
+ sql("""create table t1 using parquet as
+ select cast(null as array<binary>) c1, cast(array() as
array<binary>) c2
+ from range(10)
+ """)
+
+ checkSparkAnswerAndFallbackReason(
+ "select reverse(array(c1, c2)) AS x FROM t1",
+ fallbackReason)
+
+ checkSparkAnswerAndFallbackReason(
+ "select reverse(array(c1, c1)) AS x FROM t1",
+ fallbackReason)
+
+ checkSparkAnswerAndFallbackReason(
+ "select reverse(array(array(c1), array(c2))) AS x FROM t1",
+ fallbackReason)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
