Repository: spark Updated Branches: refs/heads/master bf7bb4977 -> 10c566cc3
[SPARK-13721][SQL] Make GeneratorOuter unresolved. ## What changes were proposed in this pull request? This is a small change to make GeneratorOuter always unresolved. It is mostly no-op change but makes it more clear since GeneratorOuter shouldn't survive analysis phase. This requires also handling in ResolveAliases rule. ## How was this patch tested? Existing generator tests. Author: Bogdan Raducanu <bog...@databricks.com> Author: Reynold Xin <r...@databricks.com> Closes #17026 from bogdanrdc/PR16958. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/10c566cc Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/10c566cc Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/10c566cc Branch: refs/heads/master Commit: 10c566cc3b5f93ddd823b1c987d9de8286cdea60 Parents: bf7bb49 Author: Bogdan Raducanu <bog...@databricks.com> Authored: Wed Feb 22 15:42:40 2017 +0100 Committer: Herman van Hovell <hvanhov...@databricks.com> Committed: Wed Feb 22 15:42:40 2017 +0100 ---------------------------------------------------------------------- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +- .../apache/spark/sql/catalyst/expressions/generators.scala | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 39a2762..c477cb4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -222,6 +222,7 @@ class Analyzer( expr.transformUp { case u @ UnresolvedAlias(child, optGenAliasFunc) => child match { case ne: NamedExpression => ne + case go @ GeneratorOuter(g: Generator) if g.resolved => MultiAlias(go, Nil) case e if !e.resolved => u case g: Generator => MultiAlias(g, Nil) case c @ Cast(ne: NamedExpression, _, _) => Alias(c, ne.name)() @@ -1665,7 +1666,6 @@ class Analyzer( var resolvedGenerator: Generate = null val newProjectList = projectList.flatMap { - case AliasedGenerator(generator, names, outer) if generator.childrenResolved => // It's a sanity check, this should not happen as the previous case will throw // exception earlier. http://git-wip-us.apache.org/repos/asf/spark/blob/10c566cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 1b98c30..e84796f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -173,7 +173,6 @@ case class Stack(children: Seq[Expression]) extends Generator { } } - /** * Only support code generation when stack produces 50 rows or less. */ @@ -204,6 +203,10 @@ case class Stack(children: Seq[Expression]) extends Generator { } } +/** + * Wrapper around another generator to specify outer behavior. This is used to implement functions + * such as explode_outer. This expression gets replaced during analysis. + */ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generator { final override def eval(input: InternalRow = null): TraversableOnce[InternalRow] = throw new UnsupportedOperationException(s"Cannot evaluate expression: $this") @@ -212,7 +215,10 @@ case class GeneratorOuter(child: Generator) extends UnaryExpression with Generat throw new UnsupportedOperationException(s"Cannot evaluate expression: $this") override def elementSchema: StructType = child.elementSchema + + override lazy val resolved: Boolean = false } + /** * A base class for [[Explode]] and [[PosExplode]]. */ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
