HIVE-19850 : Dynamic partition pruning in Tez is leading to 'No work found for tablescan' error (Ganesha Shreedhara via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e865b449 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e865b449 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e865b449 Branch: refs/heads/master-txnstats Commit: e865b449efca04b02922bbc42693a1e8f91825a9 Parents: 5e2a530 Author: Ganesha Shreedhara <ganeshashr...@gmail.com> Authored: Wed Jul 4 23:03:10 2018 -0700 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Wed Jul 4 23:03:10 2018 -0700 ---------------------------------------------------------------------- .../test/resources/testconfiguration.properties | 1 + .../hadoop/hive/ql/parse/GenTezUtils.java | 9 + ql/src/test/queries/clientpositive/dpp.q | 30 ++ .../test/results/clientpositive/llap/dpp.q.out | 371 +++++++++++++++++++ 4 files changed, 411 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index cd09481..978a806 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -491,6 +491,7 @@ minillaplocal.query.files=\ cross_prod_1.q,\ cross_prod_3.q,\ cross_prod_4.q,\ + dpp.q,\ dynamic_partition_pruning.q,\ dynamic_semijoin_reduction.q,\ dynamic_semijoin_reduction_2.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index bb0de94..dd1d6a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -276,6 +276,15 @@ public class GenTezUtils { rsToSemiJoinBranchInfo.put(rs, newSJInfo); } } + // This TableScanOperator could also be part of other events in eventOperatorSet. + for(AppMasterEventOperator event: context.eventOperatorSet) { + if(event.getConf() instanceof DynamicPruningEventDesc) { + TableScanOperator ts = ((DynamicPruningEventDesc) event.getConf()).getTableScan(); + if(ts.equals(orig)){ + ((DynamicPruningEventDesc) event.getConf()).setTableScan((TableScanOperator) newRoot); + } + } + } } context.rootToWorkMap.remove(orig); context.rootToWorkMap.put(newRoot, work); http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/test/queries/clientpositive/dpp.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/dpp.q b/ql/src/test/queries/clientpositive/dpp.q new file mode 100644 index 0000000..4bc7ccb --- /dev/null +++ b/ql/src/test/queries/clientpositive/dpp.q @@ -0,0 +1,30 @@ +set hive.tez.dynamic.partition.pruning=true; + +CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string); + +CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string); + +CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string); + + + +insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true); + +insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true); + +insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true); + + + +CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2`; + +CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3`; + +create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt; + +CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1`; + + +explain +select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt; +select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt; http://git-wip-us.apache.org/repos/asf/hive/blob/e865b449/ql/src/test/results/clientpositive/llap/dpp.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dpp.q.out b/ql/src/test/results/clientpositive/llap/dpp.q.out new file mode 100644 index 0000000..240fb4c --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/dpp.q.out @@ -0,0 +1,371 @@ +PREHOOK: query: CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t3 +POSTHOOK: query: CREATE TABLE t3(key string, value string, c_int int, c_float float, c_boolean boolean) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t3 +PREHOOK: query: insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1@dt=2018 +POSTHOOK: query: insert into table t1 partition(dt='2018') values ('k1','v1',1,1.0,true) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1@dt=2018 +POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_boolean SCRIPT [] +POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_float SCRIPT [] +POSTHOOK: Lineage: t1 PARTITION(dt=2018).c_int SCRIPT [] +POSTHOOK: Lineage: t1 PARTITION(dt=2018).key SCRIPT [] +POSTHOOK: Lineage: t1 PARTITION(dt=2018).value SCRIPT [] +PREHOOK: query: insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t2@dt=2018 +POSTHOOK: query: insert into table t2 partition(dt='2018') values ('k2','v2',2,2.0,true) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t2@dt=2018 +POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_boolean SCRIPT [] +POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_float SCRIPT [] +POSTHOOK: Lineage: t2 PARTITION(dt=2018).c_int SCRIPT [] +POSTHOOK: Lineage: t2 PARTITION(dt=2018).key SCRIPT [] +POSTHOOK: Lineage: t2 PARTITION(dt=2018).value SCRIPT [] +PREHOOK: query: insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t3@dt=2018 +POSTHOOK: query: insert into table t3 partition(dt='2018') values ('k3','v3',3,3.0,true) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t3@dt=2018 +POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_boolean SCRIPT [] +POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_float SCRIPT [] +POSTHOOK: Lineage: t3 PARTITION(dt=2018).c_int SCRIPT [] +POSTHOOK: Lineage: t3 PARTITION(dt=2018).key SCRIPT [] +POSTHOOK: Lineage: t3 PARTITION(dt=2018).value SCRIPT [] +PREHOOK: query: CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t2 +PREHOOK: Output: database:default +PREHOOK: Output: default@view1 +POSTHOOK: query: CREATE VIEW `view1` AS select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` union all select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@view1 +POSTHOOK: Lineage: view1.c_boolean EXPRESSION [(t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), ] +POSTHOOK: Lineage: view1.c_float EXPRESSION [(t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t2)t2.FieldSchema(name:c_float, type:float, comment:null), ] +POSTHOOK: Lineage: view1.c_int EXPRESSION [(t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t2)t2.FieldSchema(name:c_int, type:int, comment:null), ] +POSTHOOK: Lineage: view1.dt EXPRESSION [(t1)t1.FieldSchema(name:dt, type:string, comment:null), (t2)t2.FieldSchema(name:dt, type:string, comment:null), ] +POSTHOOK: Lineage: view1.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: view1.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3` +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t3 +PREHOOK: Output: database:default +PREHOOK: Output: default@view2 +POSTHOOK: query: CREATE VIEW `view2` AS select `t2`.`key`,`t2`.`value`,`t2`.`c_int`,`t2`.`c_float`,`t2`.`c_boolean`,`t2`.`dt` from `t2` union all select `t3`.`key`,`t3`.`value`,`t3`.`c_int`,`t3`.`c_float`,`t3`.`c_boolean`,`t3`.`dt` from `t3` +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t3 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@view2 +POSTHOOK: Lineage: view2.c_boolean EXPRESSION [(t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), (t3)t3.FieldSchema(name:c_boolean, type:boolean, comment:null), ] +POSTHOOK: Lineage: view2.c_float EXPRESSION [(t2)t2.FieldSchema(name:c_float, type:float, comment:null), (t3)t3.FieldSchema(name:c_float, type:float, comment:null), ] +POSTHOOK: Lineage: view2.c_int EXPRESSION [(t2)t2.FieldSchema(name:c_int, type:int, comment:null), (t3)t3.FieldSchema(name:c_int, type:int, comment:null), ] +POSTHOOK: Lineage: view2.dt EXPRESSION [(t2)t2.FieldSchema(name:dt, type:string, comment:null), (t3)t3.FieldSchema(name:dt, type:string, comment:null), ] +POSTHOOK: Lineage: view2.key EXPRESSION [(t2)t2.FieldSchema(name:key, type:string, comment:null), (t3)t3.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: view2.value EXPRESSION [(t2)t2.FieldSchema(name:value, type:string, comment:null), (t3)t3.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t1@dt=2018 +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t2@dt=2018 +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t3@dt=2018 +PREHOOK: Input: default@view1 +PREHOOK: Input: default@view2 +PREHOOK: Output: database:default +PREHOOK: Output: default@t4 +POSTHOOK: query: create table t4 as select key,value,c_int,c_float,c_boolean,dt from t1 union all select v1.key,v1.value,v1.c_int,v1.c_float,v1.c_boolean,v1.dt from view1 v1 join view2 v2 on v1.dt=v2.dt +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t1@dt=2018 +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t2@dt=2018 +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t3@dt=2018 +POSTHOOK: Input: default@view1 +POSTHOOK: Input: default@view2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t4 +POSTHOOK: Lineage: t4.c_boolean EXPRESSION [(t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), (t2)t2.FieldSchema(name:c_boolean, type:boolean, comment:null), ] +POSTHOOK: Lineage: t4.c_float EXPRESSION [(t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t1)t1.FieldSchema(name:c_float, type:float, comment:null), (t2)t2.FieldSchema(name:c_float, type:float, comment:null), ] +POSTHOOK: Lineage: t4.c_int EXPRESSION [(t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t1)t1.FieldSchema(name:c_int, type:int, comment:null), (t2)t2.FieldSchema(name:c_int, type:int, comment:null), ] +POSTHOOK: Lineage: t4.dt EXPRESSION [(t1)t1.FieldSchema(name:dt, type:string, comment:null), (t1)t1.FieldSchema(name:dt, type:string, comment:null), (t2)t2.FieldSchema(name:dt, type:string, comment:null), ] +POSTHOOK: Lineage: t4.key EXPRESSION [(t1)t1.FieldSchema(name:key, type:string, comment:null), (t1)t1.FieldSchema(name:key, type:string, comment:null), (t2)t2.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: t4.value EXPRESSION [(t1)t1.FieldSchema(name:value, type:string, comment:null), (t1)t1.FieldSchema(name:value, type:string, comment:null), (t2)t2.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t4 +PREHOOK: Output: database:default +PREHOOK: Output: default@view3 +POSTHOOK: query: CREATE VIEW `view3` AS select `t4`.`key`,`t4`.`value`,`t4`.`c_int`,`t4`.`c_float`,`t4`.`c_boolean`,`t4`.`dt` from `t4` union all select `t1`.`key`,`t1`.`value`,`t1`.`c_int`,`t1`.`c_float`,`t1`.`c_boolean`,`t1`.`dt` from `t1` +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t4 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@view3 +POSTHOOK: Lineage: view3.c_boolean EXPRESSION [(t4)t4.FieldSchema(name:c_boolean, type:boolean, comment:null), (t1)t1.FieldSchema(name:c_boolean, type:boolean, comment:null), ] +POSTHOOK: Lineage: view3.c_float EXPRESSION [(t4)t4.FieldSchema(name:c_float, type:float, comment:null), (t1)t1.FieldSchema(name:c_float, type:float, comment:null), ] +POSTHOOK: Lineage: view3.c_int EXPRESSION [(t4)t4.FieldSchema(name:c_int, type:int, comment:null), (t1)t1.FieldSchema(name:c_int, type:int, comment:null), ] +POSTHOOK: Lineage: view3.dt EXPRESSION [(t4)t4.FieldSchema(name:dt, type:string, comment:null), (t1)t1.FieldSchema(name:dt, type:string, comment:null), ] +POSTHOOK: Lineage: view3.key EXPRESSION [(t4)t4.FieldSchema(name:key, type:string, comment:null), (t1)t1.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: view3.value EXPRESSION [(t4)t4.FieldSchema(name:value, type:string, comment:null), (t1)t1.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: explain +select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Map 6 <- Union 7 (CONTAINS) + Map 8 <- Union 7 (CONTAINS) + Reducer 3 <- Union 2 (SIMPLE_EDGE), Union 7 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t2 + filterExpr: dt is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dt (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 5 + Map Operator Tree: + TableScan + alias: t3 + filterExpr: dt is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dt (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 6 + Map Operator Tree: + TableScan + alias: t4 + filterExpr: dt is not null (type: boolean) + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: dt is not null (type: boolean) + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: dt (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 5 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Dynamic Partitioning Event Operator + Target column: dt (string) + Target Input: t2 + Partition key expr: dt + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Map 1 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Dynamic Partitioning Event Operator + Target column: dt (string) + Target Input: t3 + Partition key expr: dt + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Map 5 + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 8 + Map Operator Tree: + TableScan + alias: t1 + filterExpr: dt is not null (type: boolean) + Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: dt (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 6 Data size: 1104 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Dynamic Partitioning Event Operator + Target column: dt (string) + Target Input: t2 + Partition key expr: dt + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Map 1 + Select Operator + expressions: _col0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 6 Data size: 2208 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Dynamic Partitioning Event Operator + Target column: dt (string) + Target Input: t3 + Partition key expr: dt + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE + Target Vertex: Map 5 + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Reducer 4 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + Union 7 + Vertex: Union 7 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Input: default@t1@dt=2018 +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t2@dt=2018 +PREHOOK: Input: default@t3 +PREHOOK: Input: default@t3@dt=2018 +PREHOOK: Input: default@t4 +PREHOOK: Input: default@view2 +PREHOOK: Input: default@view3 +#### A masked pattern was here #### +POSTHOOK: query: select count(0) from view2 v2 join view3 v3 on v2.dt=v3.dt +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Input: default@t1@dt=2018 +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t2@dt=2018 +POSTHOOK: Input: default@t3 +POSTHOOK: Input: default@t3@dt=2018 +POSTHOOK: Input: default@t4 +POSTHOOK: Input: default@view2 +POSTHOOK: Input: default@view3 +#### A masked pattern was here #### +12