This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 09222fd269 [VL] Fix FallbackTags for Delta ops (#10005)
09222fd269 is described below
commit 09222fd269f6efffeb5bb9f6644dc6eb4df41ddc
Author: Zhen Li <[email protected]>
AuthorDate: Fri Jun 20 14:16:02 2025 +0800
[VL] Fix FallbackTags for Delta ops (#10005)
* [VL] Fix FallbackTags for Delta ops.
---
.../columnar/heuristic/HeuristicTransform.scala | 4 ++++
.../scala/org/apache/gluten/execution/DeltaSuite.scala | 8 +++++++-
.../extension/columnar/enumerated/RasOffload.scala | 17 ++++++++++++++++-
3 files changed, 27 insertions(+), 2 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicTransform.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicTransform.scala
index 011453032b..b305195374 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicTransform.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicTransform.scala
@@ -20,6 +20,7 @@ import org.apache.gluten.component.Component
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.extension.caller.CallerInfo
import
org.apache.gluten.extension.columnar.ColumnarRuleApplier.ColumnarRuleCall
+import org.apache.gluten.extension.columnar.FallbackTags
import org.apache.gluten.extension.columnar.offload.OffloadSingleNode
import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode
import org.apache.gluten.extension.columnar.validator.Validator
@@ -83,6 +84,9 @@ object HeuristicTransform {
rule.offload(node)
case Validator.Failed(reason) =>
logDebug(s"Validation failed by reason: $reason on query
plan: ${node.nodeName}")
+ if (FallbackTags.maybeOffloadable(node)) {
+ FallbackTags.add(node, reason)
+ }
node
}
}
diff --git
a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
index fb135023eb..e5ee33f8d1 100644
--- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
+++ b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
@@ -209,7 +209,13 @@ abstract class DeltaSuite extends
WholeStageTransformerSuite {
s"ALTER TABLE delta.`$path` SET TBLPROPERTIES
('delta.enableDeletionVectors' = true)")
checkAnswer(spark.read.format("delta").load(path), df1.union(df2))
spark.sql(s"DELETE FROM delta.`$path` WHERE id IN
(${values2.mkString(", ")})")
- checkAnswer(spark.read.format("delta").load(path), df1)
+ import org.apache.spark.sql.execution.GlutenImplicits._
+ val df = spark.read.format("delta").load(path)
+ assert(
+ df.fallbackSummary.fallbackNodeToReason
+ .flatMap(_.values)
+ .exists(_.contains("Deletion vector is not supported in native")))
+ checkAnswer(df, df1)
}
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
index bae2f6aa45..af1fe35f7c 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
@@ -17,6 +17,7 @@
package org.apache.gluten.extension.columnar.enumerated
import org.apache.gluten.execution.{GlutenPlan, ValidatablePlan}
+import org.apache.gluten.extension.columnar.FallbackTags
import org.apache.gluten.extension.columnar.offload.OffloadSingleNode
import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode
import org.apache.gluten.extension.columnar.validator.Validator
@@ -87,7 +88,11 @@ object RasOffload {
validator.validate(node) match {
case Validator.Passed =>
case Validator.Failed(reason) =>
- // TODO: Tag the original plan with fallback reason.
+ if (FallbackTags.maybeOffloadable(node)) {
+ // FIXME: This is a solution that is not completely verified for
all cases, however
+ // it's no harm anyway so we temporarily apply this practice.
+ FallbackTags.add(node, reason)
+ }
return List.empty
}
@@ -131,6 +136,11 @@ object RasOffload {
// TODO: Tag the original plan with fallback reason. This is
a non-trivial work
// in RAS as the query plan we got here may be a copy so
may not propagate tags
// to original plan.
+ if (FallbackTags.maybeOffloadable(from)) {
+ // FIXME: This is a solution that is not completely
verified for all cases,
+ // however it's no harm anyway so we temporarily apply
this practice.
+ outComes.foreach(FallbackTags.add(from, _))
+ }
from
} else {
offloadSucceeded = true
@@ -140,6 +150,11 @@ object RasOffload {
// TODO: Tag the original plan with fallback reason. This is a
non-trivial work
// in RAS as the query plan we got here may be a copy so may
not propagate tags
// to original plan.
+ if (FallbackTags.maybeOffloadable(from)) {
+ // FIXME: This is a solution that is not completely verified
for all cases,
+ // however it's no harm anyway so we temporarily apply this
practice.
+ FallbackTags.add(from, reason)
+ }
from
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
