HIVE-18805: Add ConstantPropagate before stats annotation (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a926179f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a926179f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a926179f Branch: refs/heads/master Commit: a926179f3d17223886a77e6c9733199e5f8b2b58 Parents: 99380fb Author: Zoltan Haindrich <k...@rxd.hu> Authored: Tue Feb 27 07:48:02 2018 +0100 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Tue Feb 27 07:50:36 2018 +0100 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/TezCompiler.java | 19 ++++++-- .../llap/dynamic_semijoin_reduction_2.q.out | 2 +- .../clientpositive/llap/explainuser_1.q.out | 4 +- .../clientpositive/llap/join32_lessSize.q.out | 2 +- .../results/clientpositive/llap/join46.q.out | 8 ---- .../llap/limit_join_transpose.q.out | 48 -------------------- .../clientpositive/llap/llap_partitioned.q.out | 2 +- .../results/clientpositive/llap/mapjoin46.q.out | 4 -- .../clientpositive/llap/semijoin_hint.q.out | 16 +++---- .../llap/subquery_in_having.q.out | 8 ++-- .../clientpositive/llap/subquery_multi.q.out | 3 -- .../clientpositive/llap/subquery_notin.q.out | 3 -- .../clientpositive/llap/subquery_scalar.q.out | 16 +++---- .../clientpositive/llap/subquery_select.q.out | 4 -- .../clientpositive/llap/tez_join_tests.q.out | 4 -- .../clientpositive/llap/tez_joins_explain.q.out | 4 -- .../vector_reduce_groupby_duplicate_cols.q.out | 2 +- .../clientpositive/perf/tez/query72.q.out | 10 ++-- .../clientpositive/perf/tez/query78.q.out | 8 ++-- 19 files changed, 48 insertions(+), 119 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index f9a6386..dfd7908 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -145,6 +145,11 @@ public class TezCompiler extends TaskCompiler { runDynamicPartitionPruning(procCtx, inputs, outputs); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup dynamic partition pruning"); + // need to run this; to get consistent filterop conditions(for operator tree matching) + if (procCtx.conf.getBoolVar(ConfVars.HIVEOPTCONSTANTPROPAGATION)) { + new ConstantPropagate(ConstantPropagateOption.SHORTCUT).transform(procCtx.parseContext); + } + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); // setup stats in the operator plan runStatsAnnotation(procCtx); @@ -266,7 +271,9 @@ public class TezCompiler extends TaskCompiler { SemiJoinBranchInfo sjInfo = context.parseContext.getRsToSemiJoinBranchInfo().get(o); - if (sjInfo == null ) continue; + if (sjInfo == null ) { + continue; + } if (sjInfo.getIsHint()) { // Skipping because of hint. Mark this info, hasHint = true; @@ -866,7 +873,9 @@ public class TezCompiler extends TaskCompiler { ReduceSinkOperator rs = ((ReduceSinkOperator) child); SemiJoinBranchInfo sjInfo = pCtx.getRsToSemiJoinBranchInfo().get(rs); - if (sjInfo == null) continue; + if (sjInfo == null) { + continue; + } TableScanOperator ts = sjInfo.getTsOp(); // This is a semijoin branch. Find if this is creating a potential @@ -925,7 +934,9 @@ public class TezCompiler extends TaskCompiler { GenericUDAFBloomFilterEvaluator udafBloomFilterEvaluator = (GenericUDAFBloomFilterEvaluator) agg.getGenericUDAFEvaluator(); if (udafBloomFilterEvaluator.hasHintEntries()) + { return null; // Created using hint, skip it + } long expectedEntries = udafBloomFilterEvaluator.getExpectedEntries(); if (expectedEntries == -1 || expectedEntries > @@ -1052,7 +1063,9 @@ public class TezCompiler extends TaskCompiler { ReduceSinkOperator rs = (ReduceSinkOperator) child; SemiJoinBranchInfo sjInfo = parseContext.getRsToSemiJoinBranchInfo().get(rs); - if (sjInfo == null) continue; + if (sjInfo == null) { + continue; + } TableScanOperator ts = sjInfo.getTsOp(); if (ts != bigTableTS) { http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out index cab45c9..8bce445 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out @@ -632,7 +632,7 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/explainuser_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 43ca0d9..45b2b4f 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -2256,9 +2256,7 @@ Stage-0 PartitionCols:_col0 Group By Operator [GBY_12] (rows=83 width=87) Output:["_col0"],keys:key - Filter Operator [FIL_29] (rows=166 width=87) - predicate:(key > '2') - Please refer to the previous TableScan [TS_2] + Please refer to the previous Filter Operator [FIL_28] PREHOOK: query: explain select p_mfgr, b.p_name, p_size from part b http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index 5b5be13..f8622da 100644 --- a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -1947,7 +1947,7 @@ STAGE PLANS: 0 _col0 (type: string) 1 _col1 (type: string) outputColumnNames: _col0, _col3, _col4 - Position of Big Table: 1 + Position of Big Table: 0 Statistics: Num rows: 64 Data size: 17024 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col3 (type: string), _col0 (type: string), _col4 (type: string) http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/join46.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/join46.q.out b/ql/src/test/results/clientpositive/llap/join46.q.out index 4625b9f..ecb34d1 100644 --- a/ql/src/test/results/clientpositive/llap/join46.q.out +++ b/ql/src/test/results/clientpositive/llap/join46.q.out @@ -2086,10 +2086,6 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string) - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + @@ -2113,10 +2109,6 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string) - Select Operator - expressions: key (type: int), value (type: int), col_2 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4 Data size: 380 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: int) sort order: + http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out index 4dd7432..4db4f12 100644 --- a/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out +++ b/ql/src/test/results/clientpositive/llap/limit_join_transpose.q.out @@ -242,10 +242,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -360,10 +356,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE @@ -372,10 +364,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -518,10 +506,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE @@ -530,10 +514,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -693,10 +673,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -1133,10 +1109,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -1252,10 +1224,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Offset of rows: 1 @@ -1265,10 +1233,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -1412,10 +1376,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1 Offset of rows: 1 @@ -1425,10 +1385,6 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE TopN Hash Memory Usage: 0.1 value expressions: _col0 (type: string), _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + @@ -1591,10 +1547,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string) sort order: + http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out index ac75467..5397839 100644 --- a/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_partitioned.q.out @@ -1710,7 +1710,7 @@ STAGE PLANS: native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumnNums: [] - Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 2640 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint) outputColumnNames: _col0 http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/mapjoin46.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out index f530c1e..c6c34be 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin46.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin46.q.out @@ -1901,10 +1901,6 @@ STAGE PLANS: Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: int), _col2 (type: string) - Select Operator - expressions: key (type: int), value (type: int), col_1 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 6 Data size: 572 Basic stats: COMPLETE Column stats: COMPLETE Map Join Operator condition map: Left Outer Join 0 to 1 http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/semijoin_hint.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/semijoin_hint.q.out b/ql/src/test/results/clientpositive/llap/semijoin_hint.q.out index cdfa733..e666c50 100644 --- a/ql/src/test/results/clientpositive/llap/semijoin_hint.q.out +++ b/ql/src/test/results/clientpositive/llap/semijoin_hint.q.out @@ -487,7 +487,7 @@ STAGE PLANS: Select Operator expressions: str (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4056 Data size: 352872 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4056 Data size: 705744 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash @@ -592,7 +592,7 @@ STAGE PLANS: Select Operator expressions: cstring (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9174 Data size: 1287800 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3000) mode: hash @@ -741,7 +741,7 @@ STAGE PLANS: Select Operator expressions: str (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash @@ -1485,7 +1485,7 @@ STAGE PLANS: Select Operator expressions: _col1 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 4056 Data size: 352872 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4056 Data size: 705744 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash @@ -1962,7 +1962,7 @@ STAGE PLANS: Select Operator expressions: str (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash @@ -2381,7 +2381,7 @@ STAGE PLANS: Select Operator expressions: str (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash @@ -2562,7 +2562,7 @@ STAGE PLANS: Select Operator expressions: cstring (type: string) outputColumnNames: _col0 - Statistics: Num rows: 9174 Data size: 643900 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 9174 Data size: 1287800 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=3000) mode: hash @@ -2665,7 +2665,7 @@ STAGE PLANS: Select Operator expressions: str (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 348000 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=5000) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out index cc0b6a5..1c6c120 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_in_having.q.out @@ -1567,8 +1567,8 @@ POSTHOOK: Lineage: src_null.key SCRIPT [] POSTHOOK: Lineage: src_null.value EXPRESSION [] Warning: Map Join MAPJOIN[123][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[124][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[126][bigTable=?] in task 'Map 1' is a cross product -Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[125][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[126][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product Warning: Shuffle Join MERGEJOIN[127][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product PREHOOK: query: explain select key, value, count(*) @@ -2043,8 +2043,8 @@ STAGE PLANS: Warning: Map Join MAPJOIN[123][bigTable=?] in task 'Map 1' is a cross product Warning: Map Join MAPJOIN[124][bigTable=?] in task 'Map 1' is a cross product -Warning: Map Join MAPJOIN[126][bigTable=?] in task 'Map 1' is a cross product -Warning: Shuffle Join MERGEJOIN[125][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product +Warning: Map Join MAPJOIN[125][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[126][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 3' is a cross product Warning: Shuffle Join MERGEJOIN[127][tables = [$hdt$_2, $hdt$_3]] in Stage 'Reducer 6' is a cross product PREHOOK: query: select key, value, count(*) from src_null b http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/subquery_multi.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out index 67a22c8..de543c9 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out @@ -3989,9 +3989,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) - Filter Operator - predicate: (key > '9') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/subquery_notin.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out index 50c18c8..247d8f3 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_notin.q.out @@ -62,9 +62,6 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint), _col1 (type: bigint) - Filter Operator - predicate: (key > '2') (type: boolean) - Statistics: Num rows: 166 Data size: 14442 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator keys: key (type: string) mode: hash http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out index cec3daa..6a2c635 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_scalar.q.out @@ -1363,9 +1363,9 @@ STAGE PLANS: 1 _col2 (type: string) outputColumnNames: _col1, _col2, _col3 residual filter predicates: {((_col1 + 100) < CASE WHEN (_col3 is null) THEN (null) ELSE (_col2) END)} - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - Statistics: Num rows: 4 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() mode: hash @@ -3167,7 +3167,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col3, _col4 residual filter predicates: {(_col1 like CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END)} - Statistics: Num rows: 8 Data size: 2504 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 2472 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 @@ -3825,7 +3825,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col3, _col5, _col6 residual filter predicates: {(_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END)} - Statistics: Num rows: 7 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1 @@ -4034,7 +4034,7 @@ STAGE PLANS: 1 _col2 (type: int) outputColumnNames: _col0, _col1, _col3, _col5, _col6 residual filter predicates: {(_col1 <> CASE WHEN (_col6 is null) THEN (null) ELSE (_col5) END)} - Statistics: Num rows: 7 Data size: 140 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: int), _col3 (type: int) outputColumnNames: _col0, _col1 @@ -4378,7 +4378,7 @@ STAGE PLANS: 1 _col2 (type: string) outputColumnNames: _col0, _col1, _col2, _col3, _col4 residual filter predicates: {(not (_col1 like CASE WHEN (_col4 is null) THEN (null) ELSE (_col3) END))} - Statistics: Num rows: 7 Data size: 3535 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 3507 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col2 (type: string) outputColumnNames: _col0, _col1 @@ -6270,10 +6270,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: i (type: int), j (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/subquery_select.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/subquery_select.q.out b/ql/src/test/results/clientpositive/llap/subquery_select.q.out index 89eab37..a399d3e 100644 --- a/ql/src/test/results/clientpositive/llap/subquery_select.q.out +++ b/ql/src/test/results/clientpositive/llap/subquery_select.q.out @@ -4159,10 +4159,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: int) - Select Operator - expressions: p_partkey (type: int), p_size (type: int) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator sort order: Statistics: Num rows: 26 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out index 75879fb..dd429b6 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_tests.q.out @@ -33,10 +33,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out index 3790b37..1404e15 100644 --- a/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_joins_explain.q.out @@ -33,10 +33,6 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col1 (type: string) - Select Operator - expressions: key (type: string), value (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col1 (type: string) sort order: + http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out b/ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out index 07c1e41..d19debe 100644 --- a/ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_reduce_groupby_duplicate_cols.q.out @@ -59,7 +59,7 @@ STAGE PLANS: key expressions: 1 (type: int), 2 (type: int) sort order: ++ Map-reduce partition columns: 1 (type: int), 2 (type: int) - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs Map Vectorization: http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/perf/tez/query72.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query72.q.out b/ql/src/test/results/clientpositive/perf/tez/query72.q.out index 55ddd5a..acab54f 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query72.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query72.q.out @@ -164,7 +164,7 @@ Stage-0 <-Reducer 13 [SIMPLE_EDGE] SHUFFLE [RS_38] PartitionCols:_col4 - Merge Join Operator [MERGEJOIN_136] (rows=421645953 width=135) + Merge Join Operator [MERGEJOIN_135] (rows=421645953 width=135) Conds:RS_35._col5=RS_36._col0(Left Outer),Output:["_col1","_col4","_col6","_col7","_col9","_col10","_col16"] <-Map 19 [SIMPLE_EDGE] SHUFFLE [RS_36] @@ -176,7 +176,7 @@ Stage-0 <-Reducer 12 [SIMPLE_EDGE] SHUFFLE [RS_35] PartitionCols:_col5 - Merge Join Operator [MERGEJOIN_135] (rows=383314495 width=135) + Merge Join Operator [MERGEJOIN_134] (rows=383314495 width=135) Conds:RS_32._col3=RS_33._col0(Inner),Output:["_col1","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 18 [SIMPLE_EDGE] SHUFFLE [RS_33] @@ -190,7 +190,7 @@ Stage-0 <-Reducer 11 [SIMPLE_EDGE] SHUFFLE [RS_32] PartitionCols:_col3 - Merge Join Operator [MERGEJOIN_134] (rows=348467716 width=135) + Merge Join Operator [MERGEJOIN_133] (rows=348467716 width=135) Conds:RS_29._col2=RS_30._col0(Inner),Output:["_col1","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_30] @@ -204,7 +204,7 @@ Stage-0 <-Reducer 10 [SIMPLE_EDGE] SHUFFLE [RS_29] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_133] (rows=316788826 width=135) + Merge Join Operator [MERGEJOIN_132] (rows=316788826 width=135) Conds:RS_26._col0=RS_27._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col9","_col10"] <-Map 16 [SIMPLE_EDGE] SHUFFLE [RS_27] @@ -227,7 +227,7 @@ Stage-0 <-Reducer 2 [SIMPLE_EDGE] SHUFFLE [RS_52] PartitionCols:_col1 - Merge Join Operator [MERGEJOIN_132] (rows=41342400 width=15) + Merge Join Operator [MERGEJOIN_136] (rows=41342400 width=15) Conds:RS_49._col2=RS_50._col0(Inner),Output:["_col0","_col1","_col3","_col5"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_49] http://git-wip-us.apache.org/repos/asf/hive/blob/a926179f/ql/src/test/results/clientpositive/perf/tez/query78.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/perf/tez/query78.q.out b/ql/src/test/results/clientpositive/perf/tez/query78.q.out index 9c2d7b3..9ec4739 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query78.q.out @@ -176,7 +176,7 @@ Stage-0 Output:["_col0","_col1","_col2","_col4","_col5","_col6"] Filter Operator [FIL_58] (rows=79197206 width=135) predicate:_col8 is null - Merge Join Operator [MERGEJOIN_110] (rows=158394413 width=135) + Merge Join Operator [MERGEJOIN_108] (rows=158394413 width=135) Conds:RS_55._col2, _col3=RS_56._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] <-Map 17 [SIMPLE_EDGE] SHUFFLE [RS_55] @@ -213,7 +213,7 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_17] (rows=348477374 width=88) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_107] (rows=348477374 width=88) + Merge Join Operator [MERGEJOIN_109] (rows=348477374 width=88) Conds:RS_13._col0=RS_14._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_13] @@ -256,7 +256,7 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_38] (rows=87121617 width=135) Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(_col6)","sum(_col7)","sum(_col8)"],keys:_col4, _col3 - Merge Join Operator [MERGEJOIN_109] (rows=87121617 width=135) + Merge Join Operator [MERGEJOIN_110] (rows=87121617 width=135) Conds:RS_34._col0=RS_35._col0(Inner),Output:["_col3","_col4","_col6","_col7","_col8"] <-Map 1 [SIMPLE_EDGE] SHUFFLE [RS_34] @@ -269,7 +269,7 @@ Stage-0 Output:["_col0","_col1","_col2","_col4","_col5","_col6"] Filter Operator [FIL_32] (rows=79201469 width=135) predicate:_col8 is null - Merge Join Operator [MERGEJOIN_108] (rows=158402938 width=135) + Merge Join Operator [MERGEJOIN_107] (rows=158402938 width=135) Conds:RS_29._col1, _col3=RS_30._col0, _col1(Left Outer),Output:["_col0","_col1","_col2","_col4","_col5","_col6","_col8"] <-Map 14 [SIMPLE_EDGE] SHUFFLE [RS_29]
