This is an automated email from the ASF dual-hosted git repository.
liuneng1994 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d904cb4f45 [VL] Fix str_to_map MAP_KEY_DEDUP_POLICY comparison for
Spark 4.1 compatibility (#12035)
d904cb4f45 is described below
commit d904cb4f45cb070ad879cc3dfbea839532aaed97
Author: Surendra Singh Lilhore
<[email protected]>
AuthorDate: Tue May 5 10:49:14 2026 +0530
[VL] Fix str_to_map MAP_KEY_DEDUP_POLICY comparison for Spark 4.1
compatibility (#12035)
Starting from Spark 4.1, SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
returns an enum value instead of a String. The existing comparison against the
.toString of the EXCEPTION variant would always evaluate to not-equal
(comparing enum != String), incorrectly rejecting str_to_map offloading even
when the policy is set to EXCEPTION.
Add .toString on the left side so both operands are compared as Strings,
restoring correct behavior across all Spark versions.
What changes are proposed in this pull request?
Fix the code but to make code compatible with Spark 4.1
How was this patch tested?
Added new UT test to verify the fix.
Was this patch authored or co-authored using generative AI tooling?
GitHub Copilot CLI (powered by Claude Opus 4.6)
Related issue: #12034
---
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 5 ++++-
.../functions/ScalarFunctionsValidateSuite.scala | 26 ++++++++++++++++++++++
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index ac3e49399f..b7a1e172b2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -924,8 +924,11 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
substraitExprName: String,
children: Seq[ExpressionTransformer],
expr: Expression): ExpressionTransformer = {
+ // Calling `.toString` on both sides ensures compatibility across all
Spark versions.
+ // Starting from Spark 4.1,
`SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)` returns
+ // an enum instead of a String.
if (
- SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
+ SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY).toString
!= SQLConf.MapKeyDedupPolicy.EXCEPTION.toString
) {
GlutenExceptionUtil.throwsNotFullySupported(
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 4deff767bf..cd49c23029 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.SparkException
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.optimizer.NullPropagation
import org.apache.spark.sql.execution.ProjectExec
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
@@ -1607,4 +1608,29 @@ abstract class ScalarFunctionsValidateSuite extends
FunctionsValidateSuite {
df.collect()
}
}
+
+ test("str_to_map with MAP_KEY_DEDUP_POLICY EXCEPTION") {
+ withTempPath {
+ path =>
+ Seq("a:1,b:2,c:3", "x:10,y:20", "single:99")
+ .toDF("s")
+ .write
+ .parquet(path.getCanonicalPath)
+
+
spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("str_to_map_tbl")
+ log.info("Testing str_to_map offloading with MAP_KEY_DEDUP_POLICY set
to EXCEPTION")
+
+ withSQLConf(
+ SQLConf.MAP_KEY_DEDUP_POLICY.key ->
SQLConf.MapKeyDedupPolicy.EXCEPTION.toString) {
+ // Explicit delimiters
+ runQueryAndCompare("SELECT s, str_to_map(s, ',', ':') FROM
str_to_map_tbl") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ // Default delimiters (, and :)
+ runQueryAndCompare("SELECT s, str_to_map(s) FROM str_to_map_tbl") {
+ checkGlutenPlan[ProjectExecTransformer]
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
