This is an automated email from the ASF dual-hosted git repository.
liuneng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3c0e59f752 [CORE][VL] Handle exception in Schema validation for
unsupported join types (#11387)
3c0e59f752 is described below
commit 3c0e59f752ecf28ea85a0b4b67d0147b3832c083
Author: Surbhi-Vijay <[email protected]>
AuthorDate: Wed Jan 14 14:29:51 2026 +0530
[CORE][VL] Handle exception in Schema validation for unsupported join types
(#11387)
What changes are proposed in this pull request?
Schema validation fails during validation phase for unsupported join types.
When an unsupported join type is encountered, it throws
IllegalArgumentException . This patch catches the exception and adds validation
failure msg. This helps in falling back unsupported join type to Spark and
query successfully completes.
This issue was encountered with Spark-4.0 where new join type LeftSingle
has been introduced.
See: #11372
How was this patch tested?
Tested locally by running spark testcase with native session enabled.
---
.../apache/gluten/execution/MiscOperatorSuite.scala | 19 +++++++++++++++++++
.../org/apache/gluten/execution/ValidatablePlan.scala | 6 ++++++
2 files changed, 25 insertions(+)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index cfddfb8e21..0ea6bde3d8 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.{AnalysisException, DataFrame, Row}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper,
AQEShuffleReadExec, ShuffleQueryStageExec}
+import org.apache.spark.sql.execution.joins.BaseJoinExec
import org.apache.spark.sql.execution.window.WindowExec
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
@@ -2192,4 +2193,22 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
assert(executedPlan.count(_.isInstanceOf[ColumnarPartialProjectExec])
== 1)
}
}
+
+ testWithMinSparkVersion("Left single join should not result into exception",
"4.0") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ spark.sql("create temp view x (x1, x2) as values (1, 1), (2, 2);")
+ spark.sql("create temp view y (y1, y2) as values (2, 0), (3, -1);")
+
+ runQueryAndCompare(
+ "select *," +
+ " (select count(*) from y where x1 = y1 and y2 + 10 = x1 + 1 group
by y2) from x") {
+ df =>
+ assert(
+ getExecutedPlan(df).collect {
+ case join: BaseJoinExec if join.joinType.toString ==
"LeftSingle" => join
+ }.size == 1
+ )
+ }
+ }
+ }
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ValidatablePlan.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ValidatablePlan.scala
index 34a6c1168d..1df804ebfd 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/execution/ValidatablePlan.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/execution/ValidatablePlan.scala
@@ -85,6 +85,12 @@ trait ValidatablePlan extends GlutenPlan with LogLevelUtil {
s" consider enabling the spark.sql.legacy.allowHashOnMapType " +
s"setting to resolve this issue."
ValidationResult.failed(message)
+ case e: IllegalArgumentException =>
+ // SchemaValidation throws IllegalArgumentException in Join
validation
+ // when join type is unsupported. For example,
+ // LeftSingle join in BroadcastHashJoinExecTransformer.
+ ValidationResult.failed(
+ s"Failed to retrieve schema for ${this.nodeName}, due to:
${e.getMessage}")
}
if (!schemaValidationResult.ok()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
