HIVE-16775 : Fix HiveFilterAggregateTransposeRule when filter is always false (Pengcheng Xiong via Jesus Camacho Rodriguez)
Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b754cb7d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b754cb7d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b754cb7d Branch: refs/heads/hive-14535 Commit: b754cb7dc26fda6eb4d89a17b92cd76ca3f87b97 Parents: 9a69169 Author: Pengcheng Xiong <[email protected]> Authored: Sat Jun 3 13:08:18 2017 -0700 Committer: Ashutosh Chauhan <[email protected]> Committed: Sat Jun 3 13:08:18 2017 -0700 ---------------------------------------------------------------------- .../rules/HiveFilterAggregateTransposeRule.java | 6 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +- .../test/queries/clientpositive/filter_aggr.q | 18 + .../test/queries/clientpositive/perf/query4.q | 111 ++ .../test/queries/clientpositive/perf/query74.q | 63 + .../results/clientpositive/filter_aggr.q.out | 680 +++++++++++ .../results/clientpositive/perf/query4.q.out | 1128 ++++++++++++++++++ .../results/clientpositive/perf/query74.q.out | 529 ++++++++ 8 files changed, 2534 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java index 0e5c731..89293a7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java @@ -20,16 +20,16 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.RelFactories.FilterFactory; import org.apache.calcite.rel.rules.FilterAggregateTransposeRule; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilderFactory; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil; public class HiveFilterAggregateTransposeRule extends FilterAggregateTransposeRule { public HiveFilterAggregateTransposeRule(Class<? extends Filter> filterClass, - FilterFactory filterFactory, Class<? extends Aggregate> aggregateClass) { - super(filterClass, filterFactory, aggregateClass); + RelBuilderFactory builderFactory, Class<? extends Aggregate> aggregateClass) { + super(filterClass, builderFactory, aggregateClass); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 7f583ed..348331e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1670,7 +1670,8 @@ public class CalcitePlanner extends SemanticAnalyzer { rules.add(HiveFilterSortTransposeRule.INSTANCE); rules.add(HiveFilterJoinRule.JOIN); rules.add(HiveFilterJoinRule.FILTER_ON_JOIN); - rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, Aggregate.class)); + rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_BUILDER, + Aggregate.class)); rules.add(new FilterMergeRule(HiveRelFactories.HIVE_BUILDER)); if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_REDUCE_WITH_STATS)) { rules.add(HiveReduceExpressionsWithStatsRule.INSTANCE); http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/filter_aggr.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/filter_aggr.q b/ql/src/test/queries/clientpositive/filter_aggr.q new file mode 100644 index 0000000..44e088b --- /dev/null +++ b/ql/src/test/queries/clientpositive/filter_aggr.q @@ -0,0 +1,18 @@ +set hive.mapred.mode=nonstrict; + +explain extended +select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1; + +select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1; http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/perf/query4.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query4.q b/ql/src/test/queries/clientpositive/perf/query4.q new file mode 100644 index 0000000..631a464 --- /dev/null +++ b/ql/src/test/queries/clientpositive/perf/query4.q @@ -0,0 +1,111 @@ +set hive.mapred.mode=nonstrict; +-- start query 1 in stream 0 using template query4.tpl and seed 1819994127 +explain +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total + ,'s' sale_type + from customer + ,store_sales + ,date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2) ) year_total + ,'c' sale_type + from customer + ,catalog_sales + ,date_dim + where c_customer_sk = cs_bill_customer_sk + and cs_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year +union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2) ) year_total + ,'w' sale_type + from customer + ,web_sales + ,date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + ) + select t_s_secyear.customer_preferred_cust_flag + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_c_firstyear + ,year_total t_c_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_c_secyear.customer_id + and t_s_firstyear.customer_id = t_c_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_c_firstyear.sale_type = 'c' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_c_secyear.sale_type = 'c' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.dyear = 2001 + and t_s_secyear.dyear = 2001+1 + and t_c_firstyear.dyear = 2001 + and t_c_secyear.dyear = 2001+1 + and t_w_firstyear.dyear = 2001 + and t_w_secyear.dyear = 2001+1 + and t_s_firstyear.year_total > 0 + and t_c_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + and case when t_c_firstyear.year_total > 0 then t_c_secyear.year_total / t_c_firstyear.year_total else null end + > case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + order by t_s_secyear.customer_preferred_cust_flag +limit 100; + +-- end query 1 in stream 0 using template query4.tpl http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/queries/clientpositive/perf/query74.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/perf/query74.q b/ql/src/test/queries/clientpositive/perf/query74.q new file mode 100644 index 0000000..b25db9c --- /dev/null +++ b/ql/src/test/queries/clientpositive/perf/query74.q @@ -0,0 +1,63 @@ +set hive.mapred.mode=nonstrict; +-- start query 1 in stream 0 using template query74.tpl and seed 1556717815 +explain +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,d_year as year + ,max(ss_net_paid) year_total + ,'s' sale_type + from customer + ,store_sales + ,date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + and d_year in (2001,2001+1) + group by c_customer_id + ,c_first_name + ,c_last_name + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,d_year as year + ,max(ws_net_paid) year_total + ,'w' sale_type + from customer + ,web_sales + ,date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + and d_year in (2001,2001+1) + group by c_customer_id + ,c_first_name + ,c_last_name + ,d_year + ) + select + t_s_secyear.customer_id, t_s_secyear.customer_first_name, t_s_secyear.customer_last_name + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.year = 2001 + and t_s_secyear.year = 2001+1 + and t_w_firstyear.year = 2001 + and t_w_secyear.year = 2001+1 + and t_s_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else null end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else null end + order by 2,1,3 +limit 100; + +-- end query 1 in stream 0 using template query74.tpl http://git-wip-us.apache.org/repos/asf/hive/blob/b754cb7d/ql/src/test/results/clientpositive/filter_aggr.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/filter_aggr.q.out b/ql/src/test/results/clientpositive/filter_aggr.q.out new file mode 100644 index 0000000..db7dcae --- /dev/null +++ b/ql/src/test/results/clientpositive/filter_aggr.q.out @@ -0,0 +1,680 @@ +PREHOOK: query: explain extended +select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended +select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [null-subquery1:$hdt$_0-subquery1:src] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), 1 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + GatherStats: false + Union + Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:bigint:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + TableScan + GatherStats: false + Union + Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 251 Data size: 2666 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns _col0,_col1,_col2 + columns.types string:bigint:int + escape.delim \ + hive.serialization.extend.additional.nesting.levels true + serialization.escape.crlf true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10004 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: -mr-10005 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Truncated Path -> Alias: +#### A masked pattern was here #### + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: false (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(key) + keys: key (type: string) + mode: hash + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + tag: -1 + value expressions: _col1 (type: bigint) + auto parallelism: false + Path -> Alias: + nullscan://null/default.src/part_ [null-subquery2:$hdt$_0-subquery2:src] + Path -> Partition: + nullscan://null/default.src/part_ + Partition + input format: org.apache.hadoop.hive.ql.io.OneNullRowInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.NullStructSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.NullStructSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} + bucket_count -1 + column.name.delimiter , + columns key,value + columns.comments 'default','default' + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + nullscan://null/default.src/part_ [null-subquery2:$hdt$_0-subquery2:src] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: bigint), 2 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + column.name.delimiter , + columns _col0,_col1,_col2 + columns.types string,bigint,int + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: select key, c, m from +( +select key, c, 1 as m from (select key, count(key) as c from src group by key)s1 +union all +select key, c, 2 as m from (select key, count(key) as c from src group by key)s2 +)sub +where m = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 3 1 +10 1 1 +100 2 1 +103 2 1 +104 2 1 +105 1 1 +11 1 1 +111 1 1 +113 2 1 +114 1 1 +116 1 1 +118 2 1 +119 3 1 +12 2 1 +120 2 1 +125 2 1 +126 1 1 +128 3 1 +129 2 1 +131 1 1 +133 1 1 +134 2 1 +136 1 1 +137 2 1 +138 4 1 +143 1 1 +145 1 1 +146 2 1 +149 2 1 +15 2 1 +150 1 1 +152 2 1 +153 1 1 +155 1 1 +156 1 1 +157 1 1 +158 1 1 +160 1 1 +162 1 1 +163 1 1 +164 2 1 +165 2 1 +166 1 1 +167 3 1 +168 1 1 +169 4 1 +17 1 1 +170 1 1 +172 2 1 +174 2 1 +175 2 1 +176 2 1 +177 1 1 +178 1 1 +179 2 1 +18 2 1 +180 1 1 +181 1 1 +183 1 1 +186 1 1 +187 3 1 +189 1 1 +19 1 1 +190 1 1 +191 2 1 +192 1 1 +193 3 1 +194 1 1 +195 2 1 +196 1 1 +197 2 1 +199 3 1 +2 1 1 +20 1 1 +200 2 1 +201 1 1 +202 1 1 +203 2 1 +205 2 1 +207 2 1 +208 3 1 +209 2 1 +213 2 1 +214 1 1 +216 2 1 +217 2 1 +218 1 1 +219 2 1 +221 2 1 +222 1 1 +223 2 1 +224 2 1 +226 1 1 +228 1 1 +229 2 1 +230 5 1 +233 2 1 +235 1 1 +237 2 1 +238 2 1 +239 2 1 +24 2 1 +241 1 1 +242 2 1 +244 1 1 +247 1 1 +248 1 1 +249 1 1 +252 1 1 +255 2 1 +256 2 1 +257 1 1 +258 1 1 +26 2 1 +260 1 1 +262 1 1 +263 1 1 +265 2 1 +266 1 1 +27 1 1 +272 2 1 +273 3 1 +274 1 1 +275 1 1 +277 4 1 +278 2 1 +28 1 1 +280 2 1 +281 2 1 +282 2 1 +283 1 1 +284 1 1 +285 1 1 +286 1 1 +287 1 1 +288 2 1 +289 1 1 +291 1 1 +292 1 1 +296 1 1 +298 3 1 +30 1 1 +302 1 1 +305 1 1 +306 1 1 +307 2 1 +308 1 1 +309 2 1 +310 1 1 +311 3 1 +315 1 1 +316 3 1 +317 2 1 +318 3 1 +321 2 1 +322 2 1 +323 1 1 +325 2 1 +327 3 1 +33 1 1 +331 2 1 +332 1 1 +333 2 1 +335 1 1 +336 1 1 +338 1 1 +339 1 1 +34 1 1 +341 1 1 +342 2 1 +344 2 1 +345 1 1 +348 5 1 +35 3 1 +351 1 1 +353 2 1 +356 1 1 +360 1 1 +362 1 1 +364 1 1 +365 1 1 +366 1 1 +367 2 1 +368 1 1 +369 3 1 +37 2 1 +373 1 1 +374 1 1 +375 1 1 +377 1 1 +378 1 1 +379 1 1 +382 2 1 +384 3 1 +386 1 1 +389 1 1 +392 1 1 +393 1 1 +394 1 1 +395 2 1 +396 3 1 +397 2 1 +399 2 1 +4 1 1 +400 1 1 +401 5 1 +402 1 1 +403 3 1 +404 2 1 +406 4 1 +407 1 1 +409 3 1 +41 1 1 +411 1 1 +413 2 1 +414 2 1 +417 3 1 +418 1 1 +419 1 1 +42 2 1 +421 1 1 +424 2 1 +427 1 1 +429 2 1 +43 1 1 +430 3 1 +431 3 1 +432 1 1 +435 1 1 +436 1 1 +437 1 1 +438 3 1 +439 2 1 +44 1 1 +443 1 1 +444 1 1 +446 1 1 +448 1 1 +449 1 1 +452 1 1 +453 1 1 +454 3 1 +455 1 1 +457 1 1 +458 2 1 +459 2 1 +460 1 1 +462 2 1 +463 2 1 +466 3 1 +467 1 1 +468 4 1 +469 5 1 +47 1 1 +470 1 1 +472 1 1 +475 1 1 +477 1 1 +478 2 1 +479 1 1 +480 3 1 +481 1 1 +482 1 1 +483 1 1 +484 1 1 +485 1 1 +487 1 1 +489 4 1 +490 1 1 +491 1 1 +492 2 1 +493 1 1 +494 1 1 +495 1 1 +496 1 1 +497 1 1 +498 3 1 +5 3 1 +51 2 1 +53 1 1 +54 1 1 +57 1 1 +58 2 1 +64 1 1 +65 1 1 +66 1 1 +67 2 1 +69 1 1 +70 3 1 +72 2 1 +74 1 1 +76 2 1 +77 1 1 +78 1 1 +8 1 1 +80 1 1 +82 1 1 +83 2 1 +84 2 1 +85 1 1 +86 1 1 +87 1 1 +9 1 1 +90 3 1 +92 1 1 +95 2 1 +96 1 1 +97 2 1 +98 2 1
