Repository: spark Updated Branches: refs/heads/master 8e8633e0b -> 53ba6d6e5
[SPARK-13698][SQL] Fix Analysis Exceptions when Using Backticks in Generate ## What changes were proposed in this pull request? Analysis exception occurs while running the following query. ``` SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints` ``` ``` Failed to analyze query: org.apache.spark.sql.AnalysisException: cannot resolve '`ints`' given input columns: [a, `ints`]; line 1 pos 7 'Project ['ints] +- Generate explode(a#0.b), true, false, Some(a), [`ints`#8] +- SubqueryAlias nestedarray +- LocalRelation [a#0], [[[[1,2,3]]]] ``` ## How was this patch tested? Added new unit tests in SQLQuerySuite and HiveQlSuite Author: Dilip Biswal <dbis...@us.ibm.com> Closes #11538 from dilipbiswal/SPARK-13698. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/53ba6d6e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/53ba6d6e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/53ba6d6e Branch: refs/heads/master Commit: 53ba6d6e59d1172035c2d5e2906bb03fd1998e14 Parents: 8e8633e Author: Dilip Biswal <dbis...@us.ibm.com> Authored: Wed Mar 9 21:49:37 2016 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Wed Mar 9 21:49:37 2016 +0800 ---------------------------------------------------------------------- .../spark/sql/catalyst/parser/CatalystQl.scala | 10 ++++++++-- .../org/apache/spark/sql/hive/HiveQlSuite.scala | 20 ++++++++++++++++++++ .../sql/hive/execution/SQLQuerySuite.scala | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala index 5d96d8e..b1b449a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala @@ -899,10 +899,16 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C } val attributes = clauses.collect { - case Token(a, Nil) => UnresolvedAttribute(a.toLowerCase) + case Token(a, Nil) => UnresolvedAttribute(cleanIdentifier(a.toLowerCase)) } - Generate(generator, join = true, outer = outer, Some(alias.toLowerCase), attributes, child) + Generate( + generator, + join = true, + outer = outer, + Some(cleanIdentifier(alias.toLowerCase)), + attributes, + child) } protected def nodeToGenerator(node: ASTNode): Generator = noParseRule("Generator", node) http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala index 8e8d3f3..626550f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala @@ -208,4 +208,24 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll { |USING 'cat' AS (`thing1` int, `thing2` string) FROM `default`.`parquet_t1`) AS t """.stripMargin) } + + test("use backticks in output of Generator") { + val plan = parser.parsePlan( + """ + |SELECT `gentab2`.`gencol2` + |FROM `default`.`src` + |LATERAL VIEW explode(array(array(1, 2, 3))) `gentab1` AS `gencol1` + |LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2` + """.stripMargin) + } + + test("use escaped backticks in output of Generator") { + val plan = parser.parsePlan( + """ + |SELECT `gen``tab2`.`gen``col2` + |FROM `default`.`src` + |LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS `gen``col1` + |LATERAL VIEW explode(`gen``tab1`.`gen``col1`) `gen``tab2` AS `gen``col2` + """.stripMargin) + } } http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 0c9bac1..b42f00e 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -264,6 +264,23 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkAnswer( sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"), Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"), + Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"), + Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql( + """ + |SELECT `weird``tab`.`weird``col` + |FROM nestedArray + |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col` + """.stripMargin), + Row(1) :: Row(2) :: Row(3) :: Nil) } test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org