This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f4b7f25c34 [GLUTEN-10636][VL]Use backend validation to find all
unsupported expression (#10637)
f4b7f25c34 is described below
commit f4b7f25c34199140fbba6a72ba482b6a21f56d1d
Author: jiangjiangtian <[email protected]>
AuthorDate: Tue Nov 11 19:33:56 2025 +0800
[GLUTEN-10636][VL]Use backend validation to find all unsupported expression
(#10637)
---------
Co-authored-by: 蒋添 <[email protected]>
Co-authored-by: jiangtian <[email protected]>
---
.../backendsapi/velox/VeloxValidatorApi.scala | 26 ++++
.../execution/ColumnarPartialProjectExec.scala | 149 +++++++++++++++++++--
.../gluten/execution/MiscOperatorSuite.scala | 10 ++
cpp/velox/jni/VeloxJniWrapper.cc | 53 ++++++++
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 5 +
cpp/velox/substrait/SubstraitToVeloxPlan.h | 2 +
.../substrait/SubstraitToVeloxPlanValidator.cc | 22 +++
.../substrait/SubstraitToVeloxPlanValidator.h | 7 +
.../gluten/vectorized/NativePlanEvaluator.java | 4 +
.../gluten/vectorized/PlanEvaluatorJniWrapper.java | 8 ++
.../apache/gluten/backendsapi/ValidatorApi.scala | 10 ++
11 files changed, 287 insertions(+), 9 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
index 7b9cf91112..53892704f7 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala
@@ -18,6 +18,10 @@ package org.apache.gluten.backendsapi.velox
import org.apache.gluten.backendsapi.{BackendsApiManager, ValidatorApi}
import org.apache.gluten.execution.ValidationResult
+import org.apache.gluten.substrait.`type`.TypeNode
+import org.apache.gluten.substrait.SubstraitContext
+import org.apache.gluten.substrait.expression.ExpressionNode
+import org.apache.gluten.substrait.extensions.ExtensionBuilder
import org.apache.gluten.substrait.plan.PlanNode
import org.apache.gluten.validate.NativePlanValidationInfo
import org.apache.gluten.vectorized.NativePlanEvaluator
@@ -28,7 +32,10 @@ import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.types._
import org.apache.spark.task.TaskResources
+import io.substrait.proto.SimpleExtensionDeclaration
+
import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
class VeloxValidatorApi extends ValidatorApi {
import VeloxValidatorApi._
@@ -44,6 +51,25 @@ class VeloxValidatorApi extends ValidatorApi {
}
}
+ override def doNativeValidateExpression(
+ substraitContext: SubstraitContext,
+ expression: ExpressionNode,
+ inputTypeNode: TypeNode): Boolean = {
+ TaskResources.runUnsafe {
+ val validator =
NativePlanEvaluator.create(BackendsApiManager.getBackendName)
+ val extensionNodes =
+ new
ArrayBuffer[SimpleExtensionDeclaration](substraitContext.registeredFunction.size)
+ substraitContext.registeredFunction.forEach {
+ (key, value) =>
+ extensionNodes.append(ExtensionBuilder.makeFunctionMapping(key,
value).toProtobuf)
+ }
+ validator.doNativeValidateExpression(
+ expression.toProtobuf.toByteArray,
+ inputTypeNode.toProtobuf.toByteArray,
+ extensionNodes.map(_.toByteArray).toArray)
+ }
+ }
+
private def asValidationResult(info: NativePlanValidationInfo):
ValidationResult = {
if (info.isSupported == 1) {
return ValidationResult.succeeded
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
index 40865a0dd4..efc2a94cee 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala
@@ -19,11 +19,13 @@ package org.apache.gluten.execution
import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.columnarbatch.{ColumnarBatches, VeloxColumnarBatches}
import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.expression.{ArrowProjection, ExpressionMappings,
ExpressionUtils}
+import org.apache.gluten.expression.{ArrowProjection, ConverterUtils,
ExpressionConverter, ExpressionMappings, ExpressionUtils, TransformerState}
import org.apache.gluten.extension.columnar.transition.Convention
import org.apache.gluten.iterator.Iterators
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.sql.shims.SparkShimLoader
+import org.apache.gluten.substrait.`type`.TypeBuilder
+import org.apache.gluten.substrait.SubstraitContext
import org.apache.gluten.vectorized.{ArrowColumnarRow,
ArrowWritableColumnVector}
import org.apache.spark.rdd.RDD
@@ -35,6 +37,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric,
SQLMetrics}
import org.apache.spark.sql.hive.{HiveUDFTransformer, VeloxHiveUDFTransformer}
import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
+import scala.collection.JavaConverters._
import scala.collection.mutable.ListBuffer
/**
@@ -50,7 +53,7 @@ import scala.collection.mutable.ListBuffer
* @param child
* child plan
*/
-case class ColumnarPartialProjectExec(projectList: Seq[NamedExpression],
child: SparkPlan)(
+case class ColumnarPartialProjectExec(projectList: Seq[Expression], child:
SparkPlan)(
replacedAlias: Seq[Alias])
extends UnaryExecNode
with OrderPreservingNodeShim
@@ -95,11 +98,6 @@ case class ColumnarPartialProjectExec(projectList:
Seq[NamedExpression], child:
replacedAlias :: Nil
}
- private def validateExpression(expr: Expression): Boolean = {
- expr.deterministic && !expr.isInstanceOf[LambdaFunction] && expr.children
- .forall(validateExpression)
- }
-
private def getProjectIndexInChildOutput(exprs: Seq[Expression]): Unit = {
exprs.forall {
case a: AttributeReference =>
@@ -146,7 +144,7 @@ case class ColumnarPartialProjectExec(projectList:
Seq[NamedExpression], child:
// e.g. udf1(col) + udf2(col), it will introduce 2 cols for a2c
return ValidationResult.failed("Number of RowToColumn columns is more
than ProjectExec")
}
- if (!projectList.forall(validateExpression(_))) {
+ if (!projectList.forall(ColumnarPartialProjectExec.validateExpression)) {
return ValidationResult.failed("Contains expression not supported")
}
if (
@@ -287,6 +285,13 @@ object ColumnarPartialProjectExec {
val projectPrefix = "_SparkPartialProject"
+ val dummyPrefix = "_dummy"
+
+ def validateExpression(expr: Expression): Boolean = {
+ expr.deterministic && !expr.isInstanceOf[LambdaFunction] && expr.children
+ .forall(validateExpression)
+ }
+
/** Check if it's a hive udf but not transformable */
private def containsUnsupportedHiveUDF(h: Expression): Boolean = {
HiveUDFTransformer.isHiveUDF(h) &&
!VeloxHiveUDFTransformer.isSupportedHiveUDF(h)
@@ -358,10 +363,136 @@ object ColumnarPartialProjectExec {
}
}
+ private def doNativeValidateExpression(
+ expr: Expression,
+ replacedAlias: ListBuffer[Alias],
+ childOutput: Seq[Attribute]): Boolean = {
+ val substraitContext = new SubstraitContext
+ val output = childOutput ++ replacedAlias.map(_.toAttribute)
+ val exprTransformer =
ExpressionConverter.replaceWithExpressionTransformer(expr, output)
+ val inputTypeNodeList = TypeBuilder.makeStruct(
+ false,
+ output
+ .map(attr => ConverterUtils.getTypeNode(attr.dataType, attr.nullable))
+ .asJava)
+ BackendsApiManager.getValidatorApiInstance.doNativeValidateExpression(
+ substraitContext,
+ exprTransformer.doTransform(substraitContext),
+ inputTypeNodeList)
+ }
+
+ /**
+ * Traverse up the expression (post-order). If the function finds an
expression that is not
+ * supported by the native backend, then the function replaces the
expression by `Alias`.
+ *
+ * @return
+ * the new expression
+ */
+ private def traverseUpExpression(
+ expr: Expression,
+ replacedAlias: ListBuffer[Alias],
+ childOutput: Seq[Attribute]): Expression = {
+ val newExpr = expr.withNewChildren(expr.children.map {
+ case a: AttributeReference => a
+ case child =>
+ val newChild = child.withNewChildren(
+ child.children.map(c => traverseUpExpression(c, replacedAlias,
childOutput)))
+ // To prevent nested expressions be validated multiple times, before
doing the validation,
+ // we replace it by `Alias`.
+ val tempAttributes = new ListBuffer[Attribute]()
+ val tempChildren = child.children.zipWithIndex.map(
+ c => {
+ val child = c._1
+ val a = AttributeReference(s"$dummyPrefix${c._2}", child.dataType,
child.nullable)()
+ tempAttributes.append(a)
+ a
+ })
+ // For CreateNamedStruct, we need to create it with
`CreateStruct.apply`.
+ val toValidatedExpression = child match {
+ case CreateNamedStruct(_) => CreateStruct(tempChildren)
+ case _ => child.withNewChildren(tempChildren)
+ }
+ if (
+ !doNativeValidateExpression(
+ toValidatedExpression,
+ replacedAlias,
+ childOutput ++ tempAttributes)
+ ) {
+ replaceByAlias(newChild, replacedAlias)
+ } else {
+ newChild
+ }
+ })
+ if (!doNativeValidateExpression(newExpr, replacedAlias, childOutput)) {
+ replaceByAlias(newExpr, replacedAlias)
+ } else {
+ newExpr
+ }
+ }
+
+ private def replaceExpression(
+ expr: Expression,
+ childOutput: Seq[Attribute],
+ replacedAlias: ListBuffer[Alias]): Expression = {
+ if (expr == null) return null
+ val newExpr = replaceExpression(expr, replacedAlias)
+ if (!GlutenConfig.get.enableNativeValidation ||
!validateExpression(newExpr)) {
+ return newExpr
+ }
+ newExpr match {
+ case _: AttributeReference => newExpr
+ case alias @ Alias(child, name) =>
+ val newChild = replaceExpression(child, childOutput, replacedAlias)
+ Alias(newChild, name)(
+ alias.exprId,
+ alias.qualifier,
+ alias.explicitMetadata,
+ alias.nonInheritableMetadataKeys)
+ case x if isConditionalExpression(x) =>
+ try {
+ TransformerState.enterValidation
+ if (!doNativeValidateExpression(x, replacedAlias, childOutput)) {
+ replaceByAlias(x, replacedAlias)
+ } else {
+ x
+ }
+ } catch {
+ case _: Throwable =>
+ // If the process of conversion of the expression throws
exception, then we need to
+ // fallback the whole operator.
+ newExpr
+ } finally {
+ TransformerState.finishValidation
+ }
+ case p =>
+ try {
+ TransformerState.enterValidation
+ if (doNativeValidateExpression(p, replacedAlias, childOutput)) {
+ // Fast path: if the expression is supported by the native backend,
+ // then we don't need to traverse up the expression.
+ newExpr
+ } else {
+ // The expression is not supported by the native backend, then we
traverse down the
+ // expression to find which expression the native backend does not
support。
+ traverseUpExpression(p, replacedAlias, childOutput)
+ }
+ } catch {
+ case _: Throwable =>
+ // If the process of conversion of the expression throws
exception, then we need to
+ // fallback the whole operator. The unsupported expression may
cause the calculation
+ // crash. For example, the result data type of the expression is
decimal and the scale
+ // of the decimal is negative, but velox don't allow decimal type
with negative scale.
+ newExpr
+ } finally {
+ TransformerState.finishValidation
+ }
+ }
+ }
+
def create(original: ProjectExec): ProjectExecTransformer = {
val replacedAlias: ListBuffer[Alias] = ListBuffer()
val newProjectList = original.projectList.map {
- p => replaceExpression(p, replacedAlias).asInstanceOf[NamedExpression]
+ p => replaceExpression(p, original.child.output,
replacedAlias).asInstanceOf[NamedExpression]
}
val partialProject =
ColumnarPartialProjectExec(original.projectList,
original.child)(replacedAlias.toSeq)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index b6578c0ea5..2365425312 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -2168,4 +2168,14 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
})
}
+
+ test("Expression unsupported by backend can be handled by
ColumnarPartialProject") {
+ runQueryAndCompare(
+ "SELECT c_custkey, map_from_arrays(array(c_name), array(c_comment)) FROM
customer") {
+ df =>
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.count(_.isInstanceOf[ProjectExec]) == 0)
+ assert(executedPlan.count(_.isInstanceOf[ColumnarPartialProjectExec])
== 1)
+ }
+ }
}
diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
index f68aa54688..a969460808 100644
--- a/cpp/velox/jni/VeloxJniWrapper.cc
+++ b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -182,6 +182,59 @@
Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeValidateWithFail
JNI_METHOD_END(nullptr)
}
+JNIEXPORT jboolean JNICALL
+Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeValidateExpression(
// NOLINT
+ JNIEnv* env,
+ jobject wrapper,
+ jbyteArray exprArray,
+ jbyteArray inputTypeArray,
+ jobjectArray mappings) {
+ JNI_METHOD_START
+ auto safeExprArray = getByteArrayElementsSafe(env, exprArray);
+ auto safeInputTypeArray = getByteArrayElementsSafe(env, inputTypeArray);
+ auto exprData = safeExprArray.elems();
+ auto exprSize = env->GetArrayLength(exprArray);
+ auto inputTypeData = safeInputTypeArray.elems();
+ auto inputTypeSize = env->GetArrayLength(inputTypeArray);
+
+ ::substrait::Expression expression;
+ parseProtobuf(exprData, exprSize, &expression);
+ ::substrait::Type inputSubstraitType;
+ parseProtobuf(inputTypeData, inputTypeSize, &inputSubstraitType);
+
+ // Get the function mappings.
+ auto mappingSize = env->GetArrayLength(mappings);
+ std::unordered_map<uint64_t, std::string> functionMappings;
+ for (jsize i = 0; i < mappingSize; ++i) {
+ jbyteArray mapping = (jbyteArray)env->GetObjectArrayElement(mappings, i);
+ auto safeMappingArray = getByteArrayElementsSafe(env, mapping);
+ auto mappingData = safeMappingArray.elems();
+ auto mappingSize = env->GetArrayLength(mapping);
+
+ ::substrait::extensions::SimpleExtensionDeclaration mappingDecl;
+ parseProtobuf(mappingData, mappingSize, &mappingDecl);
+
+ const auto& sFmap = mappingDecl.extension_function();
+ auto id = sFmap.function_anchor();
+ auto name = sFmap.name();
+ functionMappings.emplace(id, name);
+ }
+
+ auto pool = defaultLeafVeloxMemoryPool().get();
+ SubstraitToVeloxPlanValidator planValidator(pool);
+ auto inputType = SubstraitParser::parseType(inputSubstraitType);
+ if (inputType->kind() != TypeKind::ROW) {
+ throw GlutenException("Input type is not a RowType.");
+ }
+ auto rowType = std::dynamic_pointer_cast<const RowType>(inputType);
+ try {
+ return planValidator.validate(expression, rowType,
std::move(functionMappings));
+ } catch (std::invalid_argument& e) {
+ return false;
+ }
+ JNI_METHOD_END(false)
+}
+
JNIEXPORT jlong JNICALL
Java_org_apache_gluten_columnarbatch_VeloxColumnarBatchJniWrapper_from( //
NOLINT
JNIEnv* env,
jobject wrapper,
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
index fc58302c14..073888a12f 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
@@ -1508,6 +1508,11 @@ void
SubstraitToVeloxPlanConverter::constructFunctionMap(const ::substrait::Plan
exprConverter_ = std::make_unique<SubstraitVeloxExprConverter>(pool_,
functionMap_);
}
+void
SubstraitToVeloxPlanConverter::constructFunctionMap(std::unordered_map<uint64_t,
std::string> substraitPlan) {
+ functionMap_ = std::move(substraitPlan);
+ exprConverter_ = std::make_unique<SubstraitVeloxExprConverter>(pool_,
functionMap_);
+}
+
std::string SubstraitToVeloxPlanConverter::findFuncSpec(uint64_t id) {
return SubstraitParser::findFunctionSpec(functionMap_, id);
}
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.h
b/cpp/velox/substrait/SubstraitToVeloxPlan.h
index ebb45a6f01..545bdae017 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlan.h
+++ b/cpp/velox/substrait/SubstraitToVeloxPlan.h
@@ -157,6 +157,8 @@ class SubstraitToVeloxPlanConverter {
/// converter based on the constructed function map.
void constructFunctionMap(const ::substrait::Plan& substraitPlan);
+ void constructFunctionMap(std::unordered_map<uint64_t, std::string>
substraitPlan);
+
/// Will return the function map used by this plan converter.
const std::unordered_map<uint64_t, std::string>& getFunctionMap() const {
return functionMap_;
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
index ae66c7e5f0..0e1ad8c683 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc
@@ -1451,4 +1451,26 @@ bool SubstraitToVeloxPlanValidator::validate(const
::substrait::Plan& plan) {
}
}
+bool SubstraitToVeloxPlanValidator::validate(
+ const ::substrait::Expression& expression,
+ const RowTypePtr& inputType,
+ std::unordered_map<uint64_t, std::string> functionMappings) {
+ try {
+ // Create plan converter and expression converter to help the validation.
+ planConverter_->constructFunctionMap(std::move(functionMappings));
+ exprConverter_ = planConverter_->getExprConverter();
+
+ if (!validateExpression(expression, inputType)) {
+ return false;
+ }
+ std::vector<core::TypedExprPtr>
expressions{exprConverter_->toVeloxExpr(expression, inputType)};
+ // Try to compile the expressions. If there is any unregistered function or
+ // mismatched type, exception will be thrown.
+ exec::ExprSet exprSet(std::move(expressions), execCtx_.get());
+ return true;
+ } catch (const VeloxException& err) {
+ return false;
+ }
+}
+
} // namespace gluten
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
index 122a6b7d4a..9005604f81 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
+++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h
@@ -17,6 +17,7 @@
#pragma once
+#include <unordered_map>
#include "SubstraitToVeloxPlan.h"
#include "velox/core/QueryCtx.h"
@@ -42,6 +43,12 @@ class SubstraitToVeloxPlanValidator {
/// Used to validate whether the computing of this Plan is supported.
bool validate(const ::substrait::Plan& plan);
+ /// Used to validate whether the computing of this Expression is supported.
+ bool validate(
+ const ::substrait::Expression& expression,
+ const RowTypePtr& inputType,
+ std::unordered_map<uint64_t, std::string> functionMappings);
+
const std::vector<std::string>& getValidateLog() const {
return validateLog_;
}
diff --git
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
index dcd0f17623..58ca86b8c3 100644
---
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
+++
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
@@ -53,6 +53,10 @@ public class NativePlanEvaluator {
return jniWrapper.nativeValidateWithFailureReason(subPlan);
}
+ public boolean doNativeValidateExpression(byte[] expression, byte[]
inputType, byte[][] mapping) {
+ return jniWrapper.nativeValidateExpression(expression, inputType, mapping);
+ }
+
public static void injectWriteFilesTempPath(String path, String fileName) {
PlanEvaluatorJniWrapper.injectWriteFilesTempPath(
path.getBytes(StandardCharsets.UTF_8),
fileName.getBytes(StandardCharsets.UTF_8));
diff --git
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
index 502bfdbcaf..9ac1c0a62b 100644
---
a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
+++
b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
@@ -51,6 +51,14 @@ public class PlanEvaluatorJniWrapper implements RuntimeAware
{
*/
native NativePlanValidationInfo nativeValidateWithFailureReason(byte[]
subPlan);
+ /**
+ * Validate the expression in native compute engine.
+ *
+ * @param expression the expression in binary format
+ * @return whether the expression is supported in native
+ */
+ native boolean nativeValidateExpression(byte[] expression, byte[] inputType,
byte[][] mapping);
+
public native String nativePlanString(byte[] substraitPlan, Boolean details);
/**
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/ValidatorApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/ValidatorApi.scala
index b259215d88..5ca80a7841 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/ValidatorApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/ValidatorApi.scala
@@ -17,6 +17,9 @@
package org.apache.gluten.backendsapi
import org.apache.gluten.execution.ValidationResult
+import org.apache.gluten.substrait.`type`.TypeNode
+import org.apache.gluten.substrait.SubstraitContext
+import org.apache.gluten.substrait.expression.ExpressionNode
import org.apache.gluten.substrait.plan.PlanNode
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
@@ -43,6 +46,13 @@ trait ValidatorApi {
/** Validate against Substrait plan node in native backend. */
def doNativeValidateWithFailureReason(plan: PlanNode): ValidationResult
+ /** Validate expression in native backend. */
+ def doNativeValidateExpression(
+ substraitContext: SubstraitContext,
+ expression: ExpressionNode,
+ inputTypeNode: TypeNode): Boolean =
+ false
+
/** Validate against Compression method, such as bzip2. */
def doCompressionSplittableValidate(compressionMethod: String): Boolean =
false
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
