This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 95ed1d4a90 [GLUTEN-12991][CORE] Remove the leftover PromotePrecision
shim and its decimal rescale path (#13002)
95ed1d4a90 is described below
commit 95ed1d4a909e61d4eba62ec5d62e2fd9fcf80080
Author: YangJie <[email protected]>
AuthorDate: Wed Sep 16 11:39:53 2026 -0400
[GLUTEN-12991][CORE] Remove the leftover PromotePrecision shim and its
decimal rescale path (#13002)
---
.../functions/MathFunctionsValidateSuite.scala | 8 ++
.../gluten/expression/ExpressionConverter.scala | 37 ++----
.../gluten/expression/ExpressionMappings.scala | 1 -
.../gluten/utils/DecimalArithmeticUtil.scala | 139 ++-------------------
.../gluten/utils/DecimalArithmeticUtilSuite.scala | 44 +++++++
.../apache/gluten/expression/ExpressionNames.scala | 1 -
.../org/apache/gluten/sql/shims/SparkShims.scala | 4 +-
.../gluten/sql/shims/spark34/Spark34Shims.scala | 7 +-
.../catalyst/expressions/PromotePrecision.scala | 34 -----
.../gluten/sql/shims/spark35/Spark35Shims.scala | 7 +-
.../catalyst/expressions/PromotePrecision.scala | 34 -----
.../gluten/sql/shims/spark40/Spark40Shims.scala | 5 -
.../catalyst/expressions/PromotePrecision.scala | 34 -----
.../gluten/sql/shims/spark41/Spark41Shims.scala | 5 -
.../catalyst/expressions/PromotePrecision.scala | 34 -----
15 files changed, 70 insertions(+), 324 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
index b9b232373b..9de39c42eb 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/MathFunctionsValidateSuite.scala
@@ -459,6 +459,14 @@ class MathFunctionsValidateSuite extends
FunctionsValidateSuite {
}
}
+ test("GLUTEN-7082: nested decimal arithmetic with a literal") {
+ runQueryAndCompare(
+ "select cast(l_orderkey as decimal(20,0)) / (cast(l_partkey as
decimal(20,0)) + 0.00001)" +
+ " from lineitem") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+
// Gluten's checkAnswer accepts any two doubles within 1e-5 of each other,
which is far
// wider than the precision loss under test here: 214.4 and
214.39999999999998 compare
// equal under it. Comparing the rendered values admits no tolerance at all,
because
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index 3cffa2cf98..f5595437bf 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -122,25 +122,21 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
expr)
}
- private def genRescaleDecimalTransformer(
+ private def genDecimalArithmeticTransformer(
substraitName: String,
b: BinaryArithmetic,
attributeSeq: Seq[Attribute],
expressionsMap: Map[Class[_], String]):
DecimalArithmeticExpressionTransformer = {
- val rescaleBinary = DecimalArithmeticUtil.rescaleLiteral(b)
- val (left, right) = DecimalArithmeticUtil.rescaleCastForDecimal(
- DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.left),
- DecimalArithmeticUtil.removeCastForDecimal(rescaleBinary.right))
val resultType = DecimalArithmeticUtil.getResultType(
b,
- left.dataType.asInstanceOf[DecimalType],
- right.dataType.asInstanceOf[DecimalType]
+ b.left.dataType.asInstanceOf[DecimalType],
+ b.right.dataType.asInstanceOf[DecimalType]
)
val leftChild =
- replaceWithExpressionTransformer0(left, attributeSeq, expressionsMap)
+ replaceWithExpressionTransformer0(b.left, attributeSeq, expressionsMap)
val rightChild =
- replaceWithExpressionTransformer0(right, attributeSeq, expressionsMap)
+ replaceWithExpressionTransformer0(b.right, attributeSeq, expressionsMap)
DecimalArithmeticExpressionTransformer(substraitName, leftChild,
rightChild, resultType, b)
}
@@ -634,12 +630,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
LiteralTransformer(m.nullOnOverflow)),
m
)
- case PromotePrecision(_ @Cast(child, _: DecimalType, _, _))
- if child.dataType
- .isInstanceOf[DecimalType] &&
!BackendsApiManager.getSettings.transformCheckOverflow =>
- replaceWithExpressionTransformer0(child, attributeSeq, expressionsMap)
- case _: NormalizeNaNAndZero | _: PromotePrecision | _: TaggingExpression
|
- _: DynamicPruningExpression =>
+ case _: NormalizeNaNAndZero | _: TaggingExpression | _:
DynamicPruningExpression =>
ChildTransformer(
substraitExprName,
replaceWithExpressionTransformer0(expr.children.head, attributeSeq,
expressionsMap),
@@ -656,18 +647,6 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
substraitExprName,
expr.children.map(replaceWithExpressionTransformer0(_, attributeSeq,
expressionsMap)),
expr)
- case CheckOverflow(b: BinaryArithmetic, decimalType, _)
- if !BackendsApiManager.getSettings.transformCheckOverflow &&
- DecimalArithmeticUtil.isDecimalArithmetic(b) =>
- val arithmeticExprName =
-
BackendsApiManager.getSparkPlanExecApiInstance.getDecimalArithmeticExprName(
- getAndCheckSubstraitName(b, expressionsMap),
- SparkShimLoader.getSparkShims.decimalAllowPrecisionLoss(b))
- val left =
- replaceWithExpressionTransformer0(b.left, attributeSeq,
expressionsMap)
- val right =
- replaceWithExpressionTransformer0(b.right, attributeSeq,
expressionsMap)
- DecimalArithmeticExpressionTransformer(arithmeticExprName, left,
right, decimalType, b)
case c: CheckOverflow =>
CheckOverflowTransformer(
substraitExprName,
@@ -688,9 +667,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
expr
)
} else {
- // Without the rescale and remove cast, result is right for high
version Spark,
- // but performance regression in velox
- genRescaleDecimalTransformer(exprName, b, attributeSeq,
expressionsMap)
+ genDecimalArithmeticTransformer(exprName, b, attributeSeq,
expressionsMap)
}
case n: NaNvl =>
BackendsApiManager.getSparkPlanExecApiInstance.genNaNvlTransformer(
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
index d58f1b5650..29c263f1f4 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
@@ -291,7 +291,6 @@ object ExpressionMappings {
Sig[DynamicPruningExpression](DYNAMIC_PRUNING_EXPRESSION),
Sig[CheckOverflow](CHECK_OVERFLOW),
Sig[MakeDecimal](MAKE_DECIMAL),
- Sig[PromotePrecision](PROMOTE_PRECISION),
Sig[SparkPartitionID](SPARK_PARTITION_ID),
Sig[AtLeastNNonNulls](AT_LEAST_N_NON_NULLS),
Sig[WidthBucket](WIDTH_BUCKET),
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala
index 893cbdd0f5..8f2fc61b8e 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/utils/DecimalArithmeticUtil.scala
@@ -19,8 +19,8 @@ package org.apache.gluten.utils
import org.apache.gluten.exception.GlutenNotSupportException
import org.apache.gluten.sql.shims.SparkShimLoader
-import org.apache.spark.sql.catalyst.expressions.{Add, BinaryArithmetic, Cast,
Divide, Expression, Literal, Multiply, Pmod, PromotePrecision, Remainder,
Subtract}
-import org.apache.spark.sql.types.{ByteType, Decimal, DecimalType,
IntegerType, LongType, ShortType}
+import org.apache.spark.sql.catalyst.expressions.{Add, BinaryArithmetic,
Divide, Multiply, Pmod, Remainder, Subtract}
+import org.apache.spark.sql.types.DecimalType
import org.apache.spark.sql.utils.DecimalTypeUtil
object DecimalArithmeticUtil {
@@ -62,6 +62,9 @@ object DecimalArithmeticUtil {
resultPrecision = intDig + decDig
resultScale = decDig
}
+ // Remainder and Pmod land here: isDecimalArithmetic admits them but no
result type is
+ // derived above. On the transformCheckOverflow path this throw is what
makes decimal % and
+ // pmod fall back.
case other =>
throw new GlutenNotSupportException(s"$other is not supported.")
}
@@ -78,9 +81,9 @@ object DecimalArithmeticUtil {
DecimalType(Math.min(precision, MAX_PRECISION), Math.min(scale, MAX_SCALE))
}
- // If casting between DecimalType, unnecessary cast is skipped to avoid data
loss,
- // because argument input type of "cast" is actually the res type of "+-*/".
- // Cast will use a wider input type, then calculates result type with less
scale than expected.
+ // Whether the expression is an arithmetic over two decimals. Remainder and
Pmod are admitted on
+ // purpose even though getResultType rejects them: dropping them here would
send both through the
+ // generic arm and offload them. See the comment on that rejection in
getResultType.
def isDecimalArithmetic(b: BinaryArithmetic): Boolean = {
if (
b.left.dataType.isInstanceOf[DecimalType] &&
@@ -92,130 +95,4 @@ object DecimalArithmeticUtil {
}
} else false
}
-
- // For decimal * 10 case, dec will be Decimal(38, 18), then the result
precision is wrong,
- // so here we will get the real precision and scale of the literal.
- private def getNewPrecisionScale(dec: Decimal): (Integer, Integer) = {
- val input = dec.abs.toJavaBigDecimal.toPlainString()
- val dotIndex = input.indexOf(".")
- if (dotIndex == -1) {
- return (input.length, 0)
- }
- if (dec.toBigDecimal.isValidLong) {
- return (dotIndex, 0)
- }
- (dec.precision, dec.scale)
- }
-
- // Change the precision and scale to the actual precision and scale of a
literal,
- // otherwise the result precision loses.
- def rescaleLiteral(arithmeticExpr: BinaryArithmetic): BinaryArithmetic = {
- if (
- arithmeticExpr.left.isInstanceOf[PromotePrecision] &&
- arithmeticExpr.right.isInstanceOf[Literal]
- ) {
- val lit = arithmeticExpr.right.asInstanceOf[Literal]
- lit.value match {
- case decLit: Decimal =>
- val (precision, scale) = getNewPrecisionScale(decLit)
- if (precision != decLit.precision || scale != decLit.scale) {
- arithmeticExpr
- .withNewChildren(Seq(arithmeticExpr.left, Cast(lit,
DecimalType(precision, scale))))
- .asInstanceOf[BinaryArithmetic]
- } else arithmeticExpr
- case _ => arithmeticExpr
- }
- } else if (
- arithmeticExpr.right.isInstanceOf[PromotePrecision]
- && arithmeticExpr.left.isInstanceOf[Literal]
- ) {
- val lit = arithmeticExpr.left.asInstanceOf[Literal]
- lit.value match {
- case decLit: Decimal =>
- val (precision, scale) = getNewPrecisionScale(decLit)
- if (precision != decLit.precision || scale != decLit.scale) {
- arithmeticExpr
- .withNewChildren(Seq(Cast(lit, DecimalType(precision, scale)),
arithmeticExpr.right))
- .asInstanceOf[BinaryArithmetic]
- } else arithmeticExpr
- case _ => arithmeticExpr
- }
- } else {
- arithmeticExpr
- }
- }
-
- // Returns whether the input expression is a combination of
PromotePrecision(Cast as DecimalType).
- private def isPromoteCast(expr: Expression): Boolean = expr match {
- case PromotePrecision(Cast(_, _: DecimalType, _, _)) => true
- case _ => false
- }
-
- def rescaleCastForDecimal(left: Expression, right: Expression): (Expression,
Expression) = {
-
- def doScale(e1: Expression, e2: Expression): (Expression, Expression) = {
- val newE2 = rescaleCastForOneSide(e2)
- val isWiderType = checkIsWiderType(
- e1.dataType.asInstanceOf[DecimalType],
- newE2.dataType.asInstanceOf[DecimalType],
- e2.dataType.asInstanceOf[DecimalType])
- if (isWiderType) (e1, newE2) else (e1, e2)
- }
-
- if (!isPromoteCast(left) && isPromoteCastIntegral(right)) {
- // Have removed PromotePrecision(Cast(DecimalType)).
- // Decimal * cast int.
- doScale(left, right)
- } else if (!isPromoteCast(right) && isPromoteCastIntegral(left)) {
- // Cast int * decimal.
- val (r, l) = doScale(right, left)
- (l, r)
- } else {
- (left, right)
- }
- }
-
- /**
- * Remove the Cast when child is PromotePrecision and PromotePrecision is
Cast(Decimal, Decimal)
- *
- * @param arithmeticExpr
- * BinaryArithmetic left or right
- * @return
- * expression removed child PromotePrecision->Cast
- */
- def removeCastForDecimal(arithmeticExpr: Expression): Expression =
arithmeticExpr match {
- case PromotePrecision(_ @Cast(child, _: DecimalType, _, _))
- if child.dataType.isInstanceOf[DecimalType] =>
- child
- case _ => arithmeticExpr
- }
-
- private def isPromoteCastIntegral(expr: Expression): Boolean = expr match {
- case PromotePrecision(_ @Cast(child, _: DecimalType, _, _)) =>
- child.dataType match {
- case IntegerType | ByteType | ShortType | LongType => true
- case _ => false
- }
- case _ => false
- }
-
- private def rescaleCastForOneSide(expr: Expression): Expression = expr match
{
- case precision @ PromotePrecision(_ @Cast(child, _: DecimalType, _, _)) =>
- child.dataType match {
- case IntegerType | ByteType | ShortType =>
- precision.withNewChildren(Seq(Cast(child, DecimalType(10, 0))))
- case LongType =>
- precision.withNewChildren(Seq(Cast(child, DecimalType(20, 0))))
- case _ => expr
- }
- case _ => expr
- }
-
- private def checkIsWiderType(
- left: DecimalType,
- right: DecimalType,
- wider: DecimalType): Boolean = {
- val widerType = SparkShimLoader.getSparkShims.widerDecimalType(left, right)
- widerType.equals(wider)
- }
}
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/utils/DecimalArithmeticUtilSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/DecimalArithmeticUtilSuite.scala
new file mode 100644
index 0000000000..2d6a32c167
--- /dev/null
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/utils/DecimalArithmeticUtilSuite.scala
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.exception.GlutenNotSupportException
+
+import org.apache.spark.sql.catalyst.expressions.{Literal, Pmod, Remainder}
+import org.apache.spark.sql.types.DecimalType
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class DecimalArithmeticUtilSuite extends AnyFunSuite {
+
+ private val decimalType = DecimalType(12, 2)
+ private val left = Literal.create(BigDecimal("1.00"), decimalType)
+ private val right = Literal.create(BigDecimal("2.00"), decimalType)
+
+ // isDecimalArithmetic admits Remainder and Pmod so that getResultType gets
a chance to reject
+ // them. On the transformCheckOverflow path that rejection is what keeps
decimal % and pmod off
+ // the native path.
+ test("remainder and pmod are admitted but have no result type") {
+ Seq(Remainder(left, right), Pmod(left, right)).foreach {
+ expr =>
+ assert(DecimalArithmeticUtil.isDecimalArithmetic(expr))
+ intercept[GlutenNotSupportException] {
+ DecimalArithmeticUtil.getResultType(expr, decimalType, decimalType)
+ }
+ }
+ }
+}
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 733adff050..5c52b52db0 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -348,7 +348,6 @@ object ExpressionNames {
final val CHECK_OVERFLOW = "check_overflow"
final val CHECK_OVERFLOW_IN_TABLE_INSERT = "check_overflow_in_table_insert"
final val MAKE_DECIMAL = "make_decimal"
- final val PROMOTE_PRECISION = "promote_precision"
final val SPARK_PARTITION_ID = "spark_partition_id"
final val MONOTONICALLY_INCREASING_ID = "monotonically_increasing_id"
final val WIDTH_BUCKET = "width_bucket"
diff --git
a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala
b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala
index 5b25a59900..d7348f233d 100644
--- a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala
@@ -36,7 +36,7 @@ import
org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike,
ShuffleExchangeLike}
import org.apache.spark.sql.execution.window.WindowGroupLimitExecShim
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{DecimalType, StringType, StructType}
+import org.apache.spark.sql.types.{StringType, StructType}
import org.apache.spark.storage.{GlutenShuffleBlockFetcherIteratorBase,
ShuffleBlockFetcherIteratorParams}
import org.apache.spark.util.SparkShimVersionUtil
@@ -184,8 +184,6 @@ trait SparkShims {
def getOtherConstantMetadataColumnValues(file: PartitionedFile):
JMap[String, Object] =
Map.empty[String, Any].asJava.asInstanceOf[JMap[String, Object]]
- def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType
-
// Spark 4.1+ (SPARK-53968) embeds allowDecimalPrecisionLoss in each
arithmetic expression's
// evalContext at analysis time. Spark41Shims overrides this to read from
the expression.
// All earlier versions have no evalContext field, so reading SQLConf.get
here is correct.
diff --git
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index fc20b60707..b16361a087 100644
---
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -23,7 +23,6 @@ import org.apache.gluten.utils.ExceptionUtils
import org.apache.spark._
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.DecimalPrecision
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.QueryPlan
@@ -40,7 +39,7 @@ import
org.apache.spark.sql.execution.exchange.BroadcastExchangeLike
import org.apache.spark.sql.extension.RewriteCreateTableAsSelect
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
-import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType,
StructField, StructType}
+import org.apache.spark.sql.types.{IntegerType, LongType, StructField,
StructType}
import org.apache.spark.storage.{GlutenShuffleBlockFetcherIterator,
GlutenShuffleBlockFetcherIteratorBase, ShuffleBlockFetcherIteratorParams}
import org.apache.hadoop.fs.{FileStatus, Path}
@@ -354,10 +353,6 @@ class Spark34Shims extends SparkShims {
}
}
- override def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType
= {
- DecimalPrecision.widerDecimalType(d1, d2)
- }
-
override def getRewriteCreateTableAsSelect(session: SparkSession):
SparkStrategy = {
RewriteCreateTableAsSelect(session)
}
diff --git
a/shims/spark34/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
b/shims/spark34/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
deleted file mode 100644
index b18a79b864..0000000000
---
a/shims/spark34/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
-import org.apache.spark.sql.types._
-
-case class PromotePrecision(child: Expression) extends UnaryExpression {
- override def dataType: DataType = child.dataType
- override def eval(input: InternalRow): Any = child.eval(input)
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
- child.genCode(ctx)
- override def prettyName: String = "promote_precision"
- override def sql: String = child.sql
- override lazy val canonicalized: Expression = child.canonicalized
-
- override protected def withNewChildInternal(newChild: Expression):
Expression =
- copy(child = newChild)
-}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 7f674ce327..585ba18aeb 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -23,7 +23,6 @@ import org.apache.gluten.sql.shims.SparkShims
import org.apache.spark._
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, InternalRow}
-import org.apache.spark.sql.catalyst.analysis.DecimalPrecision
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.QueryPlan
@@ -40,7 +39,7 @@ import
org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike,
ShuffleExchangeLike}
import org.apache.spark.sql.execution.window.{Final, GlutenFinal,
GlutenPartial, Partial, WindowGroupLimitExec, WindowGroupLimitExecShim}
import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
-import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType,
StructField, StructType}
+import org.apache.spark.sql.types.{IntegerType, LongType, StructField,
StructType}
import org.apache.spark.storage.{GlutenShuffleBlockFetcherIterator,
GlutenShuffleBlockFetcherIteratorBase, ShuffleBlockFetcherIteratorParams}
import org.apache.hadoop.fs.{FileStatus, Path}
@@ -413,10 +412,6 @@ class Spark35Shims extends SparkShims {
}
}
- override def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType
= {
- DecimalPrecision.widerDecimalType(d1, d2)
- }
-
override def getErrorMessage(raiseError: RaiseError): Option[Expression] = {
Some(raiseError.child)
}
diff --git
a/shims/spark35/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
b/shims/spark35/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
deleted file mode 100644
index b18a79b864..0000000000
---
a/shims/spark35/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
-import org.apache.spark.sql.types._
-
-case class PromotePrecision(child: Expression) extends UnaryExpression {
- override def dataType: DataType = child.dataType
- override def eval(input: InternalRow): Any = child.eval(input)
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
- child.genCode(ctx)
- override def prettyName: String = "promote_precision"
- override def sql: String = child.sql
- override lazy val canonicalized: Expression = child.canonicalized
-
- override protected def withNewChildInternal(newChild: Expression):
Expression =
- copy(child = newChild)
-}
diff --git
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
index 9ac1683dff..af563e0bc9 100644
---
a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
+++
b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala
@@ -23,7 +23,6 @@ import org.apache.gluten.sql.shims.SparkShims
import org.apache.spark._
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, InternalRow}
-import org.apache.spark.sql.catalyst.analysis.DecimalPrecisionTypeCoercion
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.{JoinType, LeftSingle}
@@ -450,10 +449,6 @@ class Spark40Shims extends SparkShims {
}
}
- override def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType
= {
- DecimalPrecisionTypeCoercion.widerDecimalType(d1, d2)
- }
-
override def getErrorMessage(raiseError: RaiseError): Option[Expression] = {
raiseError.errorParms match {
case CreateMap(children, _)
diff --git
a/shims/spark40/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
b/shims/spark40/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
deleted file mode 100644
index b18a79b864..0000000000
---
a/shims/spark40/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
-import org.apache.spark.sql.types._
-
-case class PromotePrecision(child: Expression) extends UnaryExpression {
- override def dataType: DataType = child.dataType
- override def eval(input: InternalRow): Any = child.eval(input)
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
- child.genCode(ctx)
- override def prettyName: String = "promote_precision"
- override def sql: String = child.sql
- override lazy val canonicalized: Expression = child.canonicalized
-
- override protected def withNewChildInternal(newChild: Expression):
Expression =
- copy(child = newChild)
-}
diff --git
a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
index c7cd8fed15..faf6bdd088 100644
---
a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
+++
b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala
@@ -23,7 +23,6 @@ import org.apache.gluten.sql.shims.SparkShims
import org.apache.spark._
import org.apache.spark.sql.{AnalysisException, SparkSession}
import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, InternalRow}
-import org.apache.spark.sql.catalyst.analysis.DecimalPrecisionTypeCoercion
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.plans.{JoinType, LeftSingle}
@@ -466,10 +465,6 @@ class Spark41Shims extends SparkShims {
}
}
- override def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType
= {
- DecimalPrecisionTypeCoercion.widerDecimalType(d1, d2)
- }
-
override def decimalAllowPrecisionLoss(expr: BinaryArithmetic): Boolean =
expr match {
case a: Add => a.evalContext.allowDecimalPrecisionLoss
case s: Subtract => s.evalContext.allowDecimalPrecisionLoss
diff --git
a/shims/spark41/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
b/shims/spark41/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
deleted file mode 100644
index b18a79b864..0000000000
---
a/shims/spark41/src/main/scala/org/apache/spark/sql/catalyst/expressions/PromotePrecision.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext,
ExprCode}
-import org.apache.spark.sql.types._
-
-case class PromotePrecision(child: Expression) extends UnaryExpression {
- override def dataType: DataType = child.dataType
- override def eval(input: InternalRow): Any = child.eval(input)
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode =
- child.genCode(ctx)
- override def prettyName: String = "promote_precision"
- override def sql: String = child.sql
- override lazy val canonicalized: Expression = child.canonicalized
-
- override protected def withNewChildInternal(newChild: Expression):
Expression =
- copy(child = newChild)
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
