Repository: spark
Updated Branches:
refs/heads/master 8e8633e0b -> 53ba6d6e5
[SPARK-13698][SQL] Fix Analysis Exceptions when Using Backticks in Generate
## What changes were proposed in this pull request?
Analysis exception occurs while running the following query.
```
SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`
```
```
Failed to analyze query: org.apache.spark.sql.AnalysisException: cannot resolve
'`ints`' given input columns: [a, `ints`]; line 1 pos 7
'Project ['ints]
+- Generate explode(a#0.b), true, false, Some(a), [`ints`#8]
+- SubqueryAlias nestedarray
+- LocalRelation [a#0], [[[[1,2,3]]]]
```
## How was this patch tested?
Added new unit tests in SQLQuerySuite and HiveQlSuite
Author: Dilip Biswal <dbis...@us.ibm.com>
Closes #11538 from dilipbiswal/SPARK-13698.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/53ba6d6e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/53ba6d6e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/53ba6d6e
Branch: refs/heads/master
Commit: 53ba6d6e59d1172035c2d5e2906bb03fd1998e14
Parents: 8e8633e
Author: Dilip Biswal <dbis...@us.ibm.com>
Authored: Wed Mar 9 21:49:37 2016 +0800
Committer: Wenchen Fan <wenc...@databricks.com>
Committed: Wed Mar 9 21:49:37 2016 +0800
----------------------------------------------------------------------
.../spark/sql/catalyst/parser/CatalystQl.scala | 10 ++++++++--
.../org/apache/spark/sql/hive/HiveQlSuite.scala | 20 ++++++++++++++++++++
.../sql/hive/execution/SQLQuerySuite.scala | 17 +++++++++++++++++
3 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala
index 5d96d8e..b1b449a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala
@@ -899,10 +899,16 @@
https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
}
val attributes = clauses.collect {
- case Token(a, Nil) => UnresolvedAttribute(a.toLowerCase)
+ case Token(a, Nil) => UnresolvedAttribute(cleanIdentifier(a.toLowerCase))
}
- Generate(generator, join = true, outer = outer, Some(alias.toLowerCase),
attributes, child)
+ Generate(
+ generator,
+ join = true,
+ outer = outer,
+ Some(cleanIdentifier(alias.toLowerCase)),
+ attributes,
+ child)
}
protected def nodeToGenerator(node: ASTNode): Generator =
noParseRule("Generator", node)
http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala
index 8e8d3f3..626550f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala
@@ -208,4 +208,24 @@ class HiveQlSuite extends SparkFunSuite with
BeforeAndAfterAll {
|USING 'cat' AS (`thing1` int, `thing2` string) FROM
`default`.`parquet_t1`) AS t
""".stripMargin)
}
+
+ test("use backticks in output of Generator") {
+ val plan = parser.parsePlan(
+ """
+ |SELECT `gentab2`.`gencol2`
+ |FROM `default`.`src`
+ |LATERAL VIEW explode(array(array(1, 2, 3))) `gentab1` AS `gencol1`
+ |LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2`
+ """.stripMargin)
+ }
+
+ test("use escaped backticks in output of Generator") {
+ val plan = parser.parsePlan(
+ """
+ |SELECT `gen``tab2`.`gen``col2`
+ |FROM `default`.`src`
+ |LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS
`gen``col1`
+ |LATERAL VIEW explode(`gen``tab1`.`gen``col1`) `gen``tab2` AS
`gen``col2`
+ """.stripMargin)
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/53ba6d6e/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 0c9bac1..b42f00e 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -264,6 +264,23 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
checkAnswer(
sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
Row(1) :: Row(2) :: Row(3) :: Nil)
+
+ checkAnswer(
+ sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS
`ints`"),
+ Row(1) :: Row(2) :: Row(3) :: Nil)
+
+ checkAnswer(
+ sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS
`ints`"),
+ Row(1) :: Row(2) :: Row(3) :: Nil)
+
+ checkAnswer(
+ sql(
+ """
+ |SELECT `weird``tab`.`weird``col`
+ |FROM nestedArray
+ |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
+ """.stripMargin),
+ Row(1) :: Row(2) :: Row(3) :: Nil)
}
test("SPARK-4512 Fix attribute reference resolution error when using SORT
BY") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
