Raza Jafri created SPARK-37451:
----------------------------------
Summary: Performance improvement regressed String to Decimal cast
Key: SPARK-37451
URL: https://issues.apache.org/jira/browse/SPARK-37451
Project: Spark
Issue Type: Bug
Components: Spark Core
Affects Versions: 3.2.0, 3.1.2, 3.1.1, 3.1.0, 3.1.3, 3.2.1
Reporter: Raza Jafri
A performance improvement to how Spark casts Strings to Decimal in this [PR
title|https://issues.apache.org/jira/browse/SPARK-32706], has introduced a
regression
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+--------------------+
| a|
+--------------------+
|7.836725755512218...|
+--------------------+
scala> spark.version
res2: String = 3.0.1
scala> :paste
// Entering paste mode (ctrl-D to finish)
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
spark.conf.set("spark.sql.legacy.allowNegativeScaleOfDecimal", true)
spark.conf.set("spark.rapids.sql.castStringToDecimal.enabled", true)
spark.conf.set("spark.rapids.sql.castDecimalToString.enabled", true)
val data = Seq(Row("7.836725755512218E38"))
val schema=StructType(Array(StructField("a", StringType, false)))
val df =spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.select(col("a").cast(DecimalType(37,-17))).show
// Exiting paste mode, now interpreting.
+----+
| a|
+----+
|null|
+----+
import org.apache.spark.sql.types._
import org.apache.spark.sql.Row
data: Seq[org.apache.spark.sql.Row] = List([7.836725755512218E38])
schema: org.apache.spark.sql.types.StructType =
StructType(StructField(a,StringType,false))
df: org.apache.spark.sql.DataFrame = [a: string]
scala> spark.version
res1: String = 3.1.1
The problem is that
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
