This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch branch-2 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-2 by this push: new 2bd2335 HIVE-21980: Parsing time can be high in case of deeply nested subqueries (Zoltan Haindrich reviewed by Vineet Garg) 2bd2335 is described below commit 2bd2335a8cd8677fe56d73960a4e391b2bb950bb Author: Zoltan Haindrich <k...@rxd.hu> AuthorDate: Tue Jul 16 10:50:09 2019 +0200 HIVE-21980: Parsing time can be high in case of deeply nested subqueries (Zoltan Haindrich reviewed by Vineet Garg) Signed-off-by: Zoltan Haindrich <k...@rxd.hu> (cherry picked from commit 0f39030c3d33b11ae9c14ac81e047b44e8695371) (cherry picked from commit dcfbf18cc2aa8f66c6eb7f78c7f041f7031382d2) (cherry picked from commit 6e0fc215784c56e3efc4ed325b5e2ab44b83a0eb) --- .../apache/hadoop/hive/ql/parse/FromClauseParser.g | 13 ++- .../hadoop/hive/ql/parse/TestParseDriver.java | 94 +++++++++++++++++++++- .../test/results/clientnegative/subq_insert.q.out | 2 +- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g index 00109f4..d67eed8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g @@ -98,14 +98,11 @@ fromSource atomjoinSource @init { gParent.pushMsg("joinSource", state); } @after { gParent.popMsg(state); } - : - tableSource (lateralView^)* - | - (subQuerySource) => subQuerySource (lateralView^)* - | - partitionedTableFunction (lateralView^)* - | - LPAREN! joinSource RPAREN! + : tableSource (lateralView^)* + | (LPAREN (KW_WITH|KW_SELECT|KW_MAP|KW_REDUCE|KW_FROM)) => subQuerySource (lateralView^)* + | (LPAREN LPAREN atomSelectStatement RPAREN setOperator ) => subQuerySource (lateralView^)* + | partitionedTableFunction (lateralView^)* + | LPAREN! joinSource RPAREN! ; joinSource diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java index 827921d..467f689 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestParseDriver.java @@ -19,10 +19,14 @@ package org.apache.hadoop.hive.ql.parse; import static org.junit.Assert.assertEquals; +import java.io.File; +import java.nio.charset.Charset; + import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; +import com.google.common.io.Files; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class TestParseDriver { @@ -40,7 +44,7 @@ public class TestParseDriver { String whereStr = "field5=1 and field6 in ('a', 'b')"; String havingStr = "sum(field7) > 11"; ASTNode tree = parseDriver.parse(selectStr + " from table1 where " + whereStr - + " group by field1, field2 having " + havingStr); + + " group by field1, field2 having " + havingStr); assertEquals(tree.getType(), 0); assertEquals(tree.getChildCount(), 2); ASTNode queryTree = (ASTNode) tree.getChild(0); @@ -106,7 +110,7 @@ public class TestParseDriver { assertTree((ASTNode) sumNode.getChild(1), plusNode); ASTNode tree = parseDriver.parseExpression("case when field1 = 1 then sum(field3 + field4) when field1 != 2 then " + - "sum(field3-field4) else sum(field3 * field4) end"); + "sum(field3-field4) else sum(field3 * field4) end"); assertEquals(tree.getChildCount(), 6); assertEquals(tree.getChild(0).getType(), HiveParser.KW_WHEN); assertEquals(tree.getChild(1).getType(), HiveParser.EQUAL); @@ -214,4 +218,90 @@ public class TestParseDriver { "AS test_comp_exp"); } + static class ExoticQueryBuilder { + StringBuilder sb = new StringBuilder(); + + public void recursiveSJS(int depth) { + sb.append("select "); + addColumns(30); + sb.append(" from \n"); + tablePart(depth); + sb.append(" join \n"); + tablePart(depth); + sb.append(" on ( "); + wherePart(10); + sb.append(" ) "); + sb.append(" where "); + wherePart(10); + + } + + private void tablePart(int depth) { + if (depth == 0) { + sb.append(" baseTable "); + } else { + sb.append("("); + recursiveSJS(depth - 1); + sb.append(") aa"); + } + } + + private void wherePart(int num) { + for (int i = 0; i < num - 1; i++) { + sb.append("x = "); + sb.append(i); + sb.append(" or "); + } + sb.append("x = -1"); + + } + + private void addColumns(int num) { + for (int i = 0; i < num - 1; i++) { + sb.append("c"); + sb.append(i); + sb.append(" + 2*sqrt(11)+"); + sb.append(i); + sb.append(","); + } + sb.append("cE"); + } + + public String getQuery() { + return sb.toString(); + } + } + + @Test(timeout = 10000) + public void testExoticSJSSubQuery() throws Exception { + ExoticQueryBuilder eqb = new ExoticQueryBuilder(); + eqb.recursiveSJS(10); + String q = eqb.getQuery(); + System.out.println(q); + parseDriver.parse(q); + } + + @Test + public void testJoinResulInBraces() throws Exception { + String q = + "explain select a.key, b.value from" + + "( (select key from src)a join (select value from src)b on a.key=b.value)"; + System.out.println(q); + + ASTNode root = parseDriver.parse(q); + System.out.println(root.dump()); + + } + + @Test + public void testFromSubqueryIsSetop() throws Exception { + String q = + "explain select key from ((select key from src) union (select key from src))subq "; + System.out.println(q); + + ASTNode root = parseDriver.parse(q); + System.out.println(root.dump()); + + } + } \ No newline at end of file diff --git a/ql/src/test/results/clientnegative/subq_insert.q.out b/ql/src/test/results/clientnegative/subq_insert.q.out index 620409b..8ac6815 100644 --- a/ql/src/test/results/clientnegative/subq_insert.q.out +++ b/ql/src/test/results/clientnegative/subq_insert.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10024]: Line 2:38 Cannot insert in a subquery. Inserting to table 'src1' +FAILED: ParseException line 2:32 cannot recognize input near '(' 'INSERT' 'OVERWRITE' in joinSource