This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new edcb509331 [CORE] Refactor common expression mappings out of Spark
shims (#11941)
edcb509331 is described below
commit edcb50933142863370018cfcc5a049da32644c50
Author: Mingliang Zhu <[email protected]>
AuthorDate: Wed Apr 15 23:07:15 2026 +0800
[CORE] Refactor common expression mappings out of Spark shims (#11941)
This PR refactors expression mappings after dropping Spark 3.2 support,
simplifying the codebase by moving shared mappings out of version-specific
shims and reducing duplication.
Related issue: #11379
---
.../gluten/expression/ExpressionMappings.scala | 18 +++++++++++---
.../gluten/sql/shims/spark33/Spark33Shims.scala | 28 +++-------------------
.../gluten/sql/shims/spark34/Spark34Shims.scala | 13 ----------
.../gluten/sql/shims/spark35/Spark35Shims.scala | 12 ----------
.../gluten/sql/shims/spark40/Spark40Shims.scala | 12 ----------
.../gluten/sql/shims/spark41/Spark41Shims.scala | 12 ----------
6 files changed, 18 insertions(+), 77 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index e432ac1521..d0ed9347b6 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -299,7 +299,15 @@ object ExpressionMappings {
// Decimal
Sig[UnscaledValue](UNSCALED_VALUE),
// Generator function
- Sig[Stack](STACK)
+ Sig[Stack](STACK),
+ Sig[SplitPart](SPLIT_PART),
+ Sig[Sec](SEC),
+ Sig[Csc](CSC),
+ Sig[KnownNullable](KNOWN_NULLABLE),
+ Sig[TimestampAdd](TIMESTAMP_ADD),
+ Sig[TimestampDiff](TIMESTAMP_DIFF),
+ Sig[RoundFloor](FLOOR),
+ Sig[RoundCeil](CEIL)
) ++ SparkShimLoader.getSparkShims.scalarExpressionMappings
/** Mapping Spark aggregate expression to Substrait function name */
@@ -328,7 +336,8 @@ object ExpressionMappings {
Sig[Kurtosis](KURTOSIS),
Sig[ApproximatePercentile](APPROX_PERCENTILE),
Sig[HyperLogLogPlusPlus](APPROX_COUNT_DISTINCT),
- Sig[Percentile](PERCENTILE)
+ Sig[Percentile](PERCENTILE),
+ Sig[RegrR2](REGR_R2)
) ++ SparkShimLoader.getSparkShims.aggregateExpressionMappings
/** Mapping Spark window expression to Substrait function name */
@@ -349,7 +358,10 @@ object ExpressionMappings {
Sig[NullIf](NULLIF),
Sig[Nvl](NVL),
Sig[Nvl2](NVL2),
- Sig[Right](RIGHT)
+ Sig[Right](RIGHT),
+ Sig[ArraySize](ARRAY_SIZE),
+ Sig[ILike](ILIKE),
+ Sig[MapContainsKey](MAP_CONTAINS_KEY)
) ++ SparkShimLoader.getSparkShims.runtimeReplaceableExpressionMappings
def blacklistExpressionMap: Map[Class[_], String] = {
diff --git
a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala
b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala
index c78b266270..52a9a4fd10 100644
---
a/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala
+++
b/shims/spark33/src/main/scala/org/apache/gluten/sql/shims/spark33/Spark33Shims.scala
@@ -18,7 +18,6 @@ package org.apache.gluten.sql.shims.spark33
import org.apache.gluten.execution.datasource.GlutenFormatFactory
import org.apache.gluten.expression.{ExpressionNames, Sig}
-import org.apache.gluten.expression.ExpressionNames.{CEIL, FLOOR,
KNOWN_NULLABLE, TIMESTAMP_ADD}
import org.apache.gluten.sql.shims.SparkShims
import org.apache.gluten.utils.ExceptionUtils
@@ -27,7 +26,6 @@ import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.DecimalPrecision
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.RegrR2
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
@@ -57,32 +55,12 @@ import scala.collection.mutable
class Spark33Shims extends SparkShims {
override def scalarExpressionMappings: Seq[Sig] = {
- Seq(
- Sig[SplitPart](ExpressionNames.SPLIT_PART),
- Sig[Sec](ExpressionNames.SEC),
- Sig[Csc](ExpressionNames.CSC),
- Sig[KnownNullable](KNOWN_NULLABLE),
- Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
- Sig[TimestampAdd](TIMESTAMP_ADD),
- Sig[TimestampDiff](ExpressionNames.TIMESTAMP_DIFF),
- Sig[RoundFloor](FLOOR),
- Sig[RoundCeil](CEIL)
- )
+ Seq(Sig[Empty2Null](ExpressionNames.EMPTY2NULL))
}
- override def aggregateExpressionMappings: Seq[Sig] = {
- Seq(
- Sig[RegrR2](ExpressionNames.REGR_R2)
- )
- }
+ override def aggregateExpressionMappings: Seq[Sig] = Seq()
- override def runtimeReplaceableExpressionMappings: Seq[Sig] = {
- Seq(
- Sig[ArraySize](ExpressionNames.ARRAY_SIZE),
- Sig[ILike](ExpressionNames.ILIKE),
- Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY)
- )
- }
+ override def runtimeReplaceableExpressionMappings: Seq[Sig] = Seq()
override def isNullIntolerant(expr: Expression): Boolean =
expr.isInstanceOf[NullIntolerant]
diff --git
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index 1bf51e488c..4f686271f1 100644
---
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -17,7 +17,6 @@
package org.apache.gluten.sql.shims.spark34
import org.apache.gluten.expression.{ExpressionNames, Sig}
-import org.apache.gluten.expression.ExpressionNames.KNOWN_NULLABLE
import org.apache.gluten.sql.shims.SparkShims
import org.apache.gluten.utils.ExceptionUtils
@@ -63,15 +62,7 @@ class Spark34Shims extends SparkShims {
override def scalarExpressionMappings: Seq[Sig] = {
Seq(
- Sig[SplitPart](ExpressionNames.SPLIT_PART),
- Sig[Sec](ExpressionNames.SEC),
- Sig[Csc](ExpressionNames.CSC),
- Sig[KnownNullable](KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
- Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
- Sig[TimestampDiff](ExpressionNames.TIMESTAMP_DIFF),
- Sig[RoundFloor](ExpressionNames.FLOOR),
- Sig[RoundCeil](ExpressionNames.CEIL),
Sig[Mask](ExpressionNames.MASK),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
@@ -83,7 +74,6 @@ class Spark34Shims extends SparkShims {
override def aggregateExpressionMappings: Seq[Sig] = {
Seq(
- Sig[RegrR2](ExpressionNames.REGR_R2),
Sig[RegrSlope](ExpressionNames.REGR_SLOPE),
Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT),
Sig[RegrSXY](ExpressionNames.REGR_SXY),
@@ -94,10 +84,7 @@ class Spark34Shims extends SparkShims {
override def runtimeReplaceableExpressionMappings: Seq[Sig] = {
Seq(
Sig[ArrayCompact](ExpressionNames.ARRAY_COMPACT),
- Sig[ArraySize](ExpressionNames.ARRAY_SIZE),
Sig[EqualNull](ExpressionNames.EQUAL_NULL),
- Sig[ILike](ExpressionNames.ILIKE),
- Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY),
Sig[Get](ExpressionNames.GET)
)
}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 36ba645ae1..8be204816a 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -64,16 +64,8 @@ class Spark35Shims extends SparkShims {
override def scalarExpressionMappings: Seq[Sig] = {
Seq(
- Sig[SplitPart](ExpressionNames.SPLIT_PART),
- Sig[Sec](ExpressionNames.SEC),
- Sig[Csc](ExpressionNames.CSC),
- Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[Mask](ExpressionNames.MASK),
- Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
- Sig[TimestampDiff](ExpressionNames.TIMESTAMP_DIFF),
- Sig[RoundFloor](ExpressionNames.FLOOR),
- Sig[RoundCeil](ExpressionNames.CEIL),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
Sig[ArrayAppend](ExpressionNames.ARRAY_APPEND),
@@ -85,7 +77,6 @@ class Spark35Shims extends SparkShims {
override def aggregateExpressionMappings: Seq[Sig] = {
Seq(
- Sig[RegrR2](ExpressionNames.REGR_R2),
Sig[RegrSlope](ExpressionNames.REGR_SLOPE),
Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT),
Sig[RegrSXY](ExpressionNames.REGR_SXY),
@@ -97,10 +88,7 @@ class Spark35Shims extends SparkShims {
Seq(
Sig[ArrayCompact](ExpressionNames.ARRAY_COMPACT),
Sig[ArrayPrepend](ExpressionNames.ARRAY_PREPEND),
- Sig[ArraySize](ExpressionNames.ARRAY_SIZE),
Sig[EqualNull](ExpressionNames.EQUAL_NULL),
- Sig[ILike](ExpressionNames.ILIKE),
- Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY),
Sig[Get](ExpressionNames.GET),
Sig[Luhncheck](ExpressionNames.LUHN_CHECK)
)
diff --git
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
index 0653cf04c0..5c6451a67b 100644
---
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
+++
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
@@ -67,16 +67,8 @@ class Spark40Shims extends SparkShims {
override def scalarExpressionMappings: Seq[Sig] = {
Seq(
- Sig[SplitPart](ExpressionNames.SPLIT_PART),
- Sig[Sec](ExpressionNames.SEC),
- Sig[Csc](ExpressionNames.CSC),
- Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[Mask](ExpressionNames.MASK),
- Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
- Sig[TimestampDiff](ExpressionNames.TIMESTAMP_DIFF),
- Sig[RoundFloor](ExpressionNames.FLOOR),
- Sig[RoundCeil](ExpressionNames.CEIL),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
Sig[ArrayAppend](ExpressionNames.ARRAY_APPEND),
@@ -90,7 +82,6 @@ class Spark40Shims extends SparkShims {
override def aggregateExpressionMappings: Seq[Sig] = {
Seq(
- Sig[RegrR2](ExpressionNames.REGR_R2),
Sig[RegrSlope](ExpressionNames.REGR_SLOPE),
Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT),
Sig[RegrSXY](ExpressionNames.REGR_SXY),
@@ -102,10 +93,7 @@ class Spark40Shims extends SparkShims {
Seq(
Sig[ArrayCompact](ExpressionNames.ARRAY_COMPACT),
Sig[ArrayPrepend](ExpressionNames.ARRAY_PREPEND),
- Sig[ArraySize](ExpressionNames.ARRAY_SIZE),
Sig[EqualNull](ExpressionNames.EQUAL_NULL),
- Sig[ILike](ExpressionNames.ILIKE),
- Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY),
Sig[Get](ExpressionNames.GET),
Sig[Luhncheck](ExpressionNames.LUHN_CHECK)
)
diff --git
a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
index cb953b659d..3eabf6b595 100644
---
a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
+++
b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
@@ -66,16 +66,8 @@ class Spark41Shims extends SparkShims {
override def scalarExpressionMappings: Seq[Sig] = {
Seq(
- Sig[SplitPart](ExpressionNames.SPLIT_PART),
- Sig[Sec](ExpressionNames.SEC),
- Sig[Csc](ExpressionNames.CSC),
- Sig[KnownNullable](ExpressionNames.KNOWN_NULLABLE),
Sig[Empty2Null](ExpressionNames.EMPTY2NULL),
Sig[Mask](ExpressionNames.MASK),
- Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
- Sig[TimestampDiff](ExpressionNames.TIMESTAMP_DIFF),
- Sig[RoundFloor](ExpressionNames.FLOOR),
- Sig[RoundCeil](ExpressionNames.CEIL),
Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT),
Sig[ArrayAppend](ExpressionNames.ARRAY_APPEND),
@@ -89,7 +81,6 @@ class Spark41Shims extends SparkShims {
override def aggregateExpressionMappings: Seq[Sig] = {
Seq(
- Sig[RegrR2](ExpressionNames.REGR_R2),
Sig[RegrSlope](ExpressionNames.REGR_SLOPE),
Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT),
Sig[RegrSXY](ExpressionNames.REGR_SXY),
@@ -101,10 +92,7 @@ class Spark41Shims extends SparkShims {
Seq(
Sig[ArrayCompact](ExpressionNames.ARRAY_COMPACT),
Sig[ArrayPrepend](ExpressionNames.ARRAY_PREPEND),
- Sig[ArraySize](ExpressionNames.ARRAY_SIZE),
Sig[EqualNull](ExpressionNames.EQUAL_NULL),
- Sig[ILike](ExpressionNames.ILIKE),
- Sig[MapContainsKey](ExpressionNames.MAP_CONTAINS_KEY),
Sig[Get](ExpressionNames.GET),
Sig[Luhncheck](ExpressionNames.LUHN_CHECK)
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
