This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cf3f904e95 [GLUTEN-12708][VL] Fix LATERAL VIEW OUTER stack not
consuming Velox marker, causing columnar shuffle field0 to become BOOLEAN and
crash (#12709)
cf3f904e95 is described below
commit cf3f904e95cb834c84b934d0565a8e3df2e734a8
Author: Jiaan Geng <[email protected]>
AuthorDate: Fri Aug 7 22:23:39 2026 +0800
[GLUTEN-12708][VL] Fix LATERAL VIEW OUTER stack not consuming Velox marker,
causing columnar shuffle field0 to become BOOLEAN and crash (#12709)
---
.../gluten/execution/GenerateExecTransformer.scala | 11 +++---
.../gluten/execution/MiscOperatorSuite.scala | 41 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 4 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
index 267ff3897e..8a97f488a9 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
@@ -333,10 +333,13 @@ object PullOutGenerateProjectHelper extends
PullOutProjectHelper {
}
ProjectExec(generate.requiredChildOutput ++ newOutput, newGenerate)
}
- case Explode(_) if generate.outer =>
- // Drop the last column of generatorOutput, which is the boolean
representing whether
- // the null value is unnested from the input array/map (e.g.
array(1, null)), or the
- // array/map itself is null or empty (e.g. array(), map(), null).
+ case (_: Explode | _: Stack) if generate.outer =>
+ // Drop the last column of generatorOutput, which is the boolean
marker Velox's Unnest
+ // appends for an OUTER generator (true when the row is a real
unnested value, false for
+ // the synthetic padding row of an empty/null input). Explode and
Stack share this
+ // layout -- value columns followed by the trailing marker, with no
ordinality column --
+ // so the same handling applies to both. Wrap each output column in
a CaseWhen on the
+ // marker so the padding row projects NULLs.
val isPresent =
AttributeReference(generatePostAliasName, BooleanType, nullable =
true)()
val newGenerate =
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 6cac6ccb1c..98c82217ad 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -1092,6 +1092,47 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
}
+ test("LATERAL VIEW OUTER stack followed by hash shuffle") {
+ // With OUTER, Velox's Unnest appends a trailing BOOLEAN marker column.
The Stack path has
+ // no pullOutPostProject branch to consume that marker (unlike
explode/posexplode/inline),
+ // so the native output is one column wider than the declared schema and
every upstream
+ // column shifts by one. When the exploded key drives a hash-partition
shuffle, the int32
+ // hash_partition_key that must sit at field 0 is displaced by the boolean
marker and the
+ // columnar shuffle writer aborts.
+ //
+ // The shuffle must be a *hash-partition* exchange feeding a SortMergeJoin
(not a partial
+ // aggregate, not a broadcast join) to reproduce the field-0 crash exactly
as production
+ // does: broadcasting the dim table hits a different serializer error, and
inserting a
+ // partial HashAggregate crashes earlier in the native input stream.
Disable broadcast to
+ // force the SortMergeJoin.
+ withTempView("t1_stack", "t2_dim") {
+ sql("""SELECT * from values
+ | (1, "james", 10, "lucy"),
+ | (2, "bond", 20, "lily")
+ |as tbl(id, name, id1, name1)
+ """.stripMargin).createOrReplaceTempView("t1_stack")
+ sql("""SELECT * from values
+ | (1, "a"), (2, "b"), (10, "c"), (20, "d")
+ |as tbl(k, tag)
+ """.stripMargin).createOrReplaceTempView("t2_dim")
+
+ withSQLConf("spark.sql.autoBroadcastJoinThreshold" -> "-1") {
+ runQueryAndCompare(s"""
+ |SELECT j.eq_pos, t2.tag
+ |FROM (
+ | SELECT eq_pos, val
+ | FROM t1_stack
+ | LATERAL VIEW OUTER stack(2, id, name, id1,
name1) v AS eq_pos, val
+ |) j
+ |JOIN t2_dim t2 ON j.eq_pos = t2.k
+ |ORDER BY j.eq_pos, t2.tag
+ |""".stripMargin) {
+ checkGlutenPlan[GenerateExecTransformer]
+ }
+ }
+ }
+ }
+
test("test inline function") {
Seq(true, false).foreach {
isOuter =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
