HIVE-10698 : query on view results fails with table not found error if view is created with subquery alias (CTE). (Pengcheng Xiong via Ashutosh Chauhan, John Pullokkaran)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c6a09ce3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c6a09ce3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c6a09ce3 Branch: refs/heads/llap Commit: c6a09ce30c0e9a1a6cc9eb3be37c1363997823dd Parents: 66acd26 Author: Pengcheng Xiong <[email protected]> Authored: Thu May 14 20:42:00 2015 -0700 Committer: Ashutosh Chauhan <[email protected]> Committed: Sun May 31 00:49:46 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/parse/HiveParser.g | 4 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 66 ++--- ql/src/test/queries/clientpositive/cteViews.q | 41 ++++ .../test/results/clientpositive/cteViews.q.out | 242 +++++++++++++++++++ 4 files changed, 309 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 4ec1925..b267bd2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -2127,7 +2127,7 @@ queryStatementExpression[boolean topLevel] (w=withClause {topLevel}?)? queryStatementExpressionBody[topLevel] { if ($w.tree != null) { - adaptor.addChild($queryStatementExpressionBody.tree, $w.tree); + $queryStatementExpressionBody.tree.insertChild(0, $w.tree); } } -> queryStatementExpressionBody @@ -2302,7 +2302,7 @@ selectStatementWithCTE (w=withClause)? selectStatement[true] { if ($w.tree != null) { - adaptor.addChild($selectStatement.tree, $w.tree); + $selectStatement.tree.insertChild(0, $w.tree); } } -> selectStatement http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 351c267..d609732 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -709,10 +709,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { qb.getParseInfo().setSrcForAlias(alias, tableTree); - unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase()); - if (aliasIndex != 0) { - unparseTranslator.addIdentifierTranslation((ASTNode) tabref - .getChild(aliasIndex)); + // if alias to CTE contains the alias, we do not do the translation because + // cte is actually a subquery. + if (!this.aliasToCTEs.containsKey(alias)) { + unparseTranslator.addTableNameTranslation(tableTree, SessionState.get().getCurrentDatabase()); + if (aliasIndex != 0) { + unparseTranslator.addIdentifierTranslation((ASTNode) tabref.getChild(aliasIndex)); + } } return alias; @@ -952,19 +955,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * */ private ASTNode findCTEFromName(QB qb, String cteName) { - - /* - * When saving a view definition all table references in the AST are qualified; including CTE references. - * Where as CTE definitions have no DB qualifier; so we strip out the DB qualifier before searching in - * <code>aliasToCTEs</code> map. - */ - String currDB = SessionState.get().getCurrentDatabase(); - if ( currDB != null && cteName.startsWith(currDB) && - cteName.length() > currDB.length() && - cteName.charAt(currDB.length()) == '.' ) { - cteName = cteName.substring(currDB.length() + 1); - } - StringBuffer qId = new StringBuffer(); if (qb.getId() != null) { qId.append(qb.getId()); @@ -997,14 +987,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { cteAlias = cteAlias == null ? cteName : cteAlias; ASTNode cteQryNode = findCTEFromName(qb, cteName); QBExpr cteQBExpr = new QBExpr(cteAlias); - - String cteText = ctx.getTokenRewriteStream().toString( - cteQryNode.getTokenStartIndex(), cteQryNode.getTokenStopIndex()); - final ASTNodeOrigin cteOrigin = new ASTNodeOrigin("CTE", cteName, - cteText, cteAlias, cteQryNode); - cteQryNode = (ASTNode) ParseDriver.adaptor.dupTree(cteQryNode); - SubQueryUtils.setOriginDeep(cteQryNode, cteOrigin); - doPhase1QBExpr(cteQryNode, cteQBExpr, qb.getId(), cteAlias); qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr); } @@ -1571,31 +1553,31 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { for (String alias : tabAliases) { String tab_name = qb.getTabNameForAlias(alias); + + // we first look for this alias from CTE, and then from catalog. + /* + * if this s a CTE reference: Add its AST as a SubQuery to this QB. + */ + ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase()); + if (cteNode != null) { + String cte_name = tab_name.toLowerCase(); + if (ctesExpanded.contains(cte_name)) { + throw new SemanticException("Recursive cte " + tab_name + " detected (cycle: " + + StringUtils.join(ctesExpanded, " -> ") + " -> " + tab_name + ")."); + } + addCTEAsSubQuery(qb, cte_name, alias); + sqAliasToCTEName.put(alias, cte_name); + continue; + } + Table tab = db.getTable(tab_name, false); if (tab == null) { - /* - * if this s a CTE reference: - * Add its AST as a SubQuery to this QB. - */ - ASTNode cteNode = findCTEFromName(qb, tab_name.toLowerCase()); - if ( cteNode != null ) { - String cte_name = tab_name.toLowerCase(); - if (ctesExpanded.contains(cte_name)) { - throw new SemanticException("Recursive cte " + tab_name + - " detected (cycle: " + StringUtils.join(ctesExpanded, " -> ") + - " -> " + tab_name + ")."); - } - addCTEAsSubQuery(qb, cte_name, alias); - sqAliasToCTEName.put(alias, cte_name); - continue; - } ASTNode src = qb.getParseInfo().getSrcForAlias(alias); if (null != src) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(src)); } else { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(alias)); } - } // Disallow INSERT INTO on bucketized tables http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/test/queries/clientpositive/cteViews.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/cteViews.q b/ql/src/test/queries/clientpositive/cteViews.q new file mode 100644 index 0000000..c076841 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cteViews.q @@ -0,0 +1,41 @@ +with src1 as (select key from src order by key limit 5) +select * from src1; + +use default; +drop view v; +create view v as with cte as (select key, value from src order by key limit 5) +select key from cte; + +describe extended v; + +create database bug; +use bug; +select * from default.v; +drop database bug; + +use default; +drop view v; +create view v as with cte as (select * from src order by key limit 5) +select * from cte; + +describe extended v; + +create database bug; +use bug; +select * from default.v; +drop database bug; + + +use default; +drop view v; +create view v as with src1 as (select key from src order by key limit 5) +select * from src1; + +describe extended v; + +create database bug; +use bug; +select * from default.v; +use default; +drop view v; +drop database bug; http://git-wip-us.apache.org/repos/asf/hive/blob/c6a09ce3/ql/src/test/results/clientpositive/cteViews.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/cteViews.q.out b/ql/src/test/results/clientpositive/cteViews.q.out new file mode 100644 index 0000000..6291784 --- /dev/null +++ b/ql/src/test/results/clientpositive/cteViews.q.out @@ -0,0 +1,242 @@ +PREHOOK: query: with src1 as (select key from src order by key limit 5) +select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: with src1 as (select key from src order by key limit 5) +select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 +0 +0 +10 +100 +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as with cte as (select key, value from src order by key limit 5) +select key from cte +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as with cte as (select key, value from src order by key limit 5) +select key from cte +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: describe extended v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: describe extended v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +key string + +#### A masked pattern was here #### +select key from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by key limit 5) +select `cte`.`key` from cte, tableType:VIRTUAL_VIEW) +PREHOOK: query: create database bug +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:bug +POSTHOOK: query: create database bug +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:bug +PREHOOK: query: use bug +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:bug +POSTHOOK: query: use bug +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:bug +PREHOOK: query: select * from default.v +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from default.v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +0 +0 +0 +10 +100 +PREHOOK: query: drop database bug +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:bug +PREHOOK: Output: database:bug +POSTHOOK: query: drop database bug +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:bug +POSTHOOK: Output: database:bug +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as with cte as (select * from src order by key limit 5) +select * from cte +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as with cte as (select * from src order by key limit 5) +select * from cte +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: describe extended v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: describe extended v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +key string +value string + +#### A masked pattern was here #### +select * from cte, viewExpandedText:with cte as (select `src`.`key`, `src`.`value` from `default`.`src` order by `src`.`key` limit 5) +select `cte`.`key`, `cte`.`value` from cte, tableType:VIRTUAL_VIEW) +PREHOOK: query: create database bug +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:bug +POSTHOOK: query: create database bug +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:bug +PREHOOK: query: use bug +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:bug +POSTHOOK: query: use bug +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:bug +PREHOOK: query: select * from default.v +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from default.v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +0 val_0 +0 val_0 +0 val_0 +10 val_10 +100 val_100 +PREHOOK: query: drop database bug +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:bug +PREHOOK: Output: database:bug +POSTHOOK: query: drop database bug +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:bug +POSTHOOK: Output: database:bug +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: create view v as with src1 as (select key from src order by key limit 5) +select * from src1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: create view v as with src1 as (select key from src order by key limit 5) +select * from src1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +PREHOOK: query: describe extended v +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@v +POSTHOOK: query: describe extended v +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@v +key string + +#### A masked pattern was here #### +select * from src1, viewExpandedText:with src1 as (select `src`.`key` from `default`.`src` order by key limit 5) +select `src1`.`key` from src1, tableType:VIRTUAL_VIEW) +PREHOOK: query: create database bug +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:bug +POSTHOOK: query: create database bug +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:bug +PREHOOK: query: use bug +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:bug +POSTHOOK: query: use bug +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:bug +PREHOOK: query: select * from default.v +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: select * from default.v +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@v +#### A masked pattern was here #### +0 +0 +0 +10 +100 +PREHOOK: query: use default +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:default +POSTHOOK: query: use default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:default +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: drop database bug +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:bug +PREHOOK: Output: database:bug +POSTHOOK: query: drop database bug +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:bug +POSTHOOK: Output: database:bug
