Author: namit Date: Fri Jan 27 01:05:22 2012 New Revision: 1236479 URL: http://svn.apache.org/viewvc?rev=1236479&view=rev Log: HIVE-2754 NPE in union with lateral view (Yongqiang He via namit)
Added: hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java?rev=1236479&r1=1236478&r2=1236479&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Fri Jan 27 01:05:22 2012 @@ -63,7 +63,17 @@ public class GenMRRedSink3 implements No Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx .getMapCurrCtx(); GenMapRedCtx mapredCtx = mapCurrCtx.get(ctx.getCurrUnionOp()); - Task<? extends Serializable> unionTask = mapredCtx.getCurrTask(); + Task<? extends Serializable> unionTask = null; + if(mapredCtx != null) { + unionTask = mapredCtx.getCurrTask(); + } else { + Operator<? extends Serializable> topOp = ctx.getCurrTopOp(); + if(topOp == null) { + return null; + } + unionTask = ctx.getCurrTask(); + } + MapredWork plan = (MapredWork) unionTask.getWork(); HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> opTaskMap = ctx .getOpTaskMap(); Added: hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q?rev=1236479&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q Fri Jan 27 01:05:22 2012 @@ -0,0 +1,43 @@ + +create table test_union_lateral_view(key int, arr_ele int, value string); + +EXPLAIN +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +; + +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +; + +select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20; Added: hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out?rev=1236479&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out Fri Jan 27 01:05:22 2012 @@ -0,0 +1,342 @@ +PREHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@test_union_lateral_view +PREHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_TABLE_OR_COL arr)) arr_ele (TOK_TABALIAS c))) (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION array 1 2 3) arr)))) (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION array 1 2 3) arr)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (= (TOK_TABLE_OR_COL hr) '12')))))) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) arr_ele) arr_ele) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_T ABLE_OR_COL a) value) value)))) d) (TOK_TABREF (TOK_TABNAME src) b) (= (. (TOK_TABLE_OR_COL d) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME test_union_lateral_view))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL d) arr_ele)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL d) value))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + b + TableScan + alias: b + Reduce Output Operator + key expressions: + expr: key + type: string + sort order: + + Map-reduce partition columns: + expr: key + type: string + tag: 1 + value expressions: + expr: key + type: string + d-subquery1:a-subquery1:src + TableScan + alias: src + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: array(1,2,3) + type: array<int> + outputColumnNames: _col0, _col1, _col2 + Union + Lateral View Forward + Select Operator + SELECT * : (no compute) + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: + expr: _col3 + type: int + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col1 + type: string + sort order: + + Map-reduce partition columns: + expr: _col1 + type: string + tag: 0 + value expressions: + expr: _col0 + type: int + expr: _col2 + type: string + Select Operator + expressions: + expr: _col2 + type: array<int> + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: + expr: _col3 + type: int + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col1 + type: string + sort order: + + Map-reduce partition columns: + expr: _col1 + type: string + tag: 0 + value expressions: + expr: _col0 + type: int + expr: _col2 + type: string + d-subquery2:a-subquery2:srcpart + TableScan + alias: srcpart + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: array(1,2,3) + type: array<int> + outputColumnNames: _col0, _col1, _col2 + Union + Lateral View Forward + Select Operator + SELECT * : (no compute) + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: + expr: _col3 + type: int + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col1 + type: string + sort order: + + Map-reduce partition columns: + expr: _col1 + type: string + tag: 0 + value expressions: + expr: _col0 + type: int + expr: _col2 + type: string + Select Operator + expressions: + expr: _col2 + type: array<int> + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2, _col3 + Select Operator + expressions: + expr: _col3 + type: int + expr: _col0 + type: string + expr: _col1 + type: string + outputColumnNames: _col0, _col1, _col2 + Reduce Output Operator + key expressions: + expr: _col1 + type: string + sort order: + + Map-reduce partition columns: + expr: _col1 + type: string + tag: 0 + value expressions: + expr: _col0 + type: int + expr: _col2 + type: string + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + condition expressions: + 0 {VALUE._col0} {VALUE._col2} + 1 {VALUE._col0} + handleSkewJoin: false + outputColumnNames: _col0, _col2, _col3 + Select Operator + expressions: + expr: _col3 + type: string + expr: _col0 + type: int + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: UDFToInteger(_col0) + type: int + expr: _col1 + type: int + expr: _col2 + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_union_lateral_view + + Stage: Stage-0 + Move Operator + tables: + replace: true + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_union_lateral_view + + Stage: Stage-2 + Stats-Aggr Operator + + +PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@test_union_lateral_view +POSTHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view +SELECT b.key, d.arr_ele, d.value +FROM ( + SELECT c.arr_ele as arr_ele, a.key as key, a.value as value + FROM ( + SELECT key, value, array(1,2,3) as arr + FROM src + + UNION ALL + + SELECT key, value, array(1,2,3) as arr + FROM srcpart + WHERE ds = '2008-04-08' and hr='12' + ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele +) d +LEFT OUTER JOIN src b +ON d.key = b.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@test_union_lateral_view +POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION [] +POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@test_union_lateral_view +#### A masked pattern was here #### +POSTHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_union_lateral_view +#### A masked pattern was here #### +POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION [] +POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default), ] +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 1 val_0 +0 2 val_0 +0 2 val_0