cloud-fan commented on code in PR #45350:
URL: https://github.com/apache/spark/pull/45350#discussion_r1515378900
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala:
##########
@@ -2876,28 +2876,36 @@ class Analyzer(override val catalogManager:
CatalogManager) extends RuleExecutor
}
}
+ // We must wait until all expressions except for generator functions are
resolved before
+ // rewriting generator functions in Project/Aggregate. This is necessary
to make this rule
+ // stable for different execution orders of analyzer rules. See also
SPARK-47241.
+ private def canRewriteGenerator(namedExprs: Seq[NamedExpression]): Boolean
= {
+ namedExprs.forall { ne =>
+ ne.resolved || {
+ trimNonTopLevelAliases(ne) match {
+ case AliasedGenerator(_, _, _) => true
+ case _ => false
+ }
+ }
+ }
+ }
+
def apply(plan: LogicalPlan): LogicalPlan =
plan.resolveOperatorsUpWithPruning(
_.containsPattern(GENERATOR), ruleId) {
case Project(projectList, _) if projectList.exists(hasNestedGenerator) =>
val nestedGenerator = projectList.find(hasNestedGenerator).get
throw
QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))
- case Project(projectList, _) if projectList.count(hasGenerator) > 1 =>
- val generators = projectList.filter(hasGenerator).map(trimAlias)
- throw QueryCompilationErrors.moreThanOneGeneratorError(generators,
"SELECT")
-
case Aggregate(_, aggList, _) if aggList.exists(hasNestedGenerator) =>
val nestedGenerator = aggList.find(hasNestedGenerator).get
throw
QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))
case Aggregate(_, aggList, _) if aggList.count(hasGenerator) > 1 =>
val generators = aggList.filter(hasGenerator).map(trimAlias)
- throw QueryCompilationErrors.moreThanOneGeneratorError(generators,
"aggregate")
+ throw QueryCompilationErrors.moreThanOneGeneratorError(generators)
Review Comment:
Another reason is we can't always figure out if it's aggregate or not. If
there is no GROUP BY, the plan is still `Project` and we may fail before
analyzer rewrite it to `Aggregate`, then we report `SELECT clause` anyway.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
