Repository: hive Updated Branches: refs/heads/master ddc246528 -> 61372dfaf
HIVE-20331: Query with union all, lateral view and Join fails with "cannot find parent in the child operator" (Aihua Xu, reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/61372dfa Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/61372dfa Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/61372dfa Branch: refs/heads/master Commit: 61372dfafa35b43be710554e4920e3b6ed4dacdf Parents: ddc2465 Author: Aihua Xu <aihu...@apache.org> Authored: Wed Aug 8 09:07:31 2018 -0700 Committer: Aihua Xu <aihu...@apache.org> Committed: Thu Aug 16 10:57:21 2018 -0700 ---------------------------------------------------------------------- .../hive/ql/optimizer/GenMRProcContext.java | 11 ++ .../hive/ql/optimizer/GenMRTableScan1.java | 2 + .../clientpositive/unionall_lateralview.q | 29 +++++ .../clientpositive/unionall_lateralview.q.out | 105 +++++++++++++++++++ 4 files changed, 147 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java index f80395d..782ce16 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java @@ -219,6 +219,17 @@ public class GenMRProcContext implements NodeProcessorCtx { } /** + * The context is reused across the rules. Reset so the following info is not + * incorrectly carried over to the following optimizations starting with the new TS. + */ + public void reset() { + currTask = null; + currTopOp = null; + currUnionOp = null; + currAliasId = null; + } + + /** * @return reducer to task mapping */ public HashMap<Operator<? extends OperatorDesc>, http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 6295d7f..bb53ce8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -66,6 +66,8 @@ public class GenMRTableScan1 implements NodeProcessor { Object... nodeOutputs) throws SemanticException { TableScanOperator op = (TableScanOperator) nd; GenMRProcContext ctx = (GenMRProcContext) opProcCtx; + ctx.reset(); + ParseContext parseCtx = ctx.getParseCtx(); Table table = op.getConf().getTableMetadata(); Class<? extends InputFormat> inputFormat = table.getInputFormatClass(); http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/test/queries/clientpositive/unionall_lateralview.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/unionall_lateralview.q b/ql/src/test/queries/clientpositive/unionall_lateralview.q new file mode 100644 index 0000000..457f404 --- /dev/null +++ b/ql/src/test/queries/clientpositive/unionall_lateralview.q @@ -0,0 +1,29 @@ +-- Test the case HIVE-20331 with the union all, lateral view and join +DROP TABLE IF EXISTS unionall_lateralview1; +DROP TABLE IF EXISTS unionall_lateralview2; +CREATE TABLE unionall_lateralview1(col1 INT); +INSERT INTO unionall_lateralview1 VALUES(1), (2); +CREATE TABLE unionall_lateralview2(col1 INT); + +INSERT INTO unionall_lateralview2 +SELECT 1 AS `col1` +FROM unionall_lateralview1 +UNION ALL + SELECT 2 AS `col1` + FROM + (SELECT col1 + FROM unionall_lateralview1 + ) x1 + JOIN + (SELECT col1 + FROM + (SELECT + Row_Number() over (PARTITION BY col1 ORDER BY col1) AS `col1` + FROM unionall_lateralview1 + ) x2 lateral VIEW explode(map(10,1))`mapObj` AS `col2`, `col3` + ) `expdObj`; + +SELECT * FROM unionall_lateralview2 ORDER BY col1; + +DROP TABLE unionall_lateralview1; +DROP TABLE unionall_lateralview2; http://git-wip-us.apache.org/repos/asf/hive/blob/61372dfa/ql/src/test/results/clientpositive/unionall_lateralview.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/unionall_lateralview.q.out b/ql/src/test/results/clientpositive/unionall_lateralview.q.out new file mode 100644 index 0000000..db64777 --- /dev/null +++ b/ql/src/test/results/clientpositive/unionall_lateralview.q.out @@ -0,0 +1,105 @@ +PREHOOK: query: DROP TABLE IF EXISTS unionall_lateralview1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS unionall_lateralview1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS unionall_lateralview2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS unionall_lateralview2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE unionall_lateralview1(col1 INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unionall_lateralview1 +POSTHOOK: query: CREATE TABLE unionall_lateralview1(col1 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unionall_lateralview1 +PREHOOK: query: INSERT INTO unionall_lateralview1 VALUES(1), (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@unionall_lateralview1 +POSTHOOK: query: INSERT INTO unionall_lateralview1 VALUES(1), (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@unionall_lateralview1 +POSTHOOK: Lineage: unionall_lateralview1.col1 SCRIPT [] +PREHOOK: query: CREATE TABLE unionall_lateralview2(col1 INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@unionall_lateralview2 +POSTHOOK: query: CREATE TABLE unionall_lateralview2(col1 INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@unionall_lateralview2 +Warning: Shuffle Join JOIN[18][tables = [x1, expdobj]] in Stage 'Stage-9:MAPRED' is a cross product +PREHOOK: query: INSERT INTO unionall_lateralview2 +SELECT 1 AS `col1` +FROM unionall_lateralview1 +UNION ALL + SELECT 2 AS `col1` + FROM + (SELECT col1 + FROM unionall_lateralview1 + ) x1 + JOIN + (SELECT col1 + FROM + (SELECT + Row_Number() over (PARTITION BY col1 ORDER BY col1) AS `col1` + FROM unionall_lateralview1 + ) x2 lateral VIEW explode(map(10,1))`mapObj` AS `col2`, `col3` + ) `expdObj` +PREHOOK: type: QUERY +PREHOOK: Input: default@unionall_lateralview1 +PREHOOK: Output: default@unionall_lateralview2 +POSTHOOK: query: INSERT INTO unionall_lateralview2 +SELECT 1 AS `col1` +FROM unionall_lateralview1 +UNION ALL + SELECT 2 AS `col1` + FROM + (SELECT col1 + FROM unionall_lateralview1 + ) x1 + JOIN + (SELECT col1 + FROM + (SELECT + Row_Number() over (PARTITION BY col1 ORDER BY col1) AS `col1` + FROM unionall_lateralview1 + ) x2 lateral VIEW explode(map(10,1))`mapObj` AS `col2`, `col3` + ) `expdObj` +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unionall_lateralview1 +POSTHOOK: Output: default@unionall_lateralview2 +POSTHOOK: Lineage: unionall_lateralview2.col1 EXPRESSION [] +PREHOOK: query: SELECT * FROM unionall_lateralview2 ORDER BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@unionall_lateralview2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM unionall_lateralview2 ORDER BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@unionall_lateralview2 +#### A masked pattern was here #### +1 +1 +2 +2 +2 +2 +PREHOOK: query: DROP TABLE unionall_lateralview1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unionall_lateralview1 +PREHOOK: Output: default@unionall_lateralview1 +POSTHOOK: query: DROP TABLE unionall_lateralview1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unionall_lateralview1 +POSTHOOK: Output: default@unionall_lateralview1 +PREHOOK: query: DROP TABLE unionall_lateralview2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@unionall_lateralview2 +PREHOOK: Output: default@unionall_lateralview2 +POSTHOOK: query: DROP TABLE unionall_lateralview2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@unionall_lateralview2 +POSTHOOK: Output: default@unionall_lateralview2
