liujiayi771 commented on code in PR #4952:
URL: https://github.com/apache/incubator-gluten/pull/4952#discussion_r1530193158
##########
backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala:
##########
@@ -750,4 +750,8 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
CHGenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan =
generate
Review Comment:
The logic of pull out should also apply to CH? It's just that CH hasn't
encountered a scenario that requires pull out yet?
##########
backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala:
##########
@@ -659,4 +662,81 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
GenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = {
+ if (supportsGenerate(generate)) {
+ val newGeneratorChild =
generate.generator.asInstanceOf[UnaryExpression].child match {
Review Comment:
Some `Generator`s are not `UnaryExpression`, use match case, and throw
exception if it is not `UnaryExpression`.
##########
backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala:
##########
@@ -750,4 +750,8 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
CHGenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan =
generate
Review Comment:
The logic of pull out should also apply to CH? It's just that CH hasn't
encountered a scenario that requires pull out yet? If that is the case, then we
can centralize these pieces of code within the pull out rule.
##########
backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala:
##########
@@ -659,4 +662,81 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
GenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = {
+ if (supportsGenerate(generate)) {
+ val newGeneratorChild =
generate.generator.asInstanceOf[UnaryExpression].child match {
+ case attr: Attribute => attr
+ case expr @ (Literal(_, _) | CreateMap(_, _) | CreateArray(_, _)) =>
+ Alias(expr, generatePreAliasName)()
Review Comment:
It would be better to maintain consistency with the code patterns of other
'pull out' operations, using methods such as `isNotAttribute` and
`replaceExpressionWithAttribute` from `PullOutProjectHelper`.
##########
backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala:
##########
@@ -659,4 +662,81 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
GenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = {
+ if (supportsGenerate(generate)) {
+ val newGeneratorChild =
generate.generator.asInstanceOf[UnaryExpression].child match {
+ case attr: Attribute => attr
+ case expr @ (Literal(_, _) | CreateMap(_, _) | CreateArray(_, _)) =>
+ Alias(expr, generatePreAliasName)()
+ case other =>
+ throw new UnsupportedOperationException(
+ s"Fail to execute ${generate.generator.getClass.getSimpleName} " +
+ s"with child type ${other.getClass.getSimpleName}")
+ }
+ generate.copy(
+ generator =
+
generate.generator.withNewChildren(Seq(newGeneratorChild)).asInstanceOf[Generator],
+ child = ProjectExec(generate.child.output :+ newGeneratorChild,
generate.child)
Review Comment:
Use `eliminateProjectList` to get the `projectList` of pre project.
##########
backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/SparkPlanExecApiImpl.scala:
##########
@@ -659,4 +662,81 @@ class SparkPlanExecApiImpl extends SparkPlanExecApi {
): GenerateExecTransformerBase = {
GenerateExecTransformer(generator, requiredChildOutput, outer,
generatorOutput, child)
}
+
+ override def genPreProjectForGenerate(generate: GenerateExec): SparkPlan = {
+ if (supportsGenerate(generate)) {
+ val newGeneratorChild =
generate.generator.asInstanceOf[UnaryExpression].child match {
+ case attr: Attribute => attr
+ case expr @ (Literal(_, _) | CreateMap(_, _) | CreateArray(_, _)) =>
+ Alias(expr, generatePreAliasName)()
+ case other =>
+ throw new UnsupportedOperationException(
Review Comment:
Are these the only three scenarios that require a 'pull out'? Is it the case
that in other situations, as long as it's not an Attribute, a 'pull out' can be
performed without necessarily throwing an exception?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]