Repository: hive Updated Branches: refs/heads/master 656a99e7e -> 10bd23eb5
http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index a7ae10e..1107f82 100644 --- a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -200,9 +200,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### str _c1 +line3 4501 line1 4501 line2 4501 -line3 4501 PREHOOK: query: SELECT strct.B, count(val) FROM orc_create_complex GROUP BY strct.B PREHOOK: type: QUERY PREHOOK: Input: default@orc_create_complex @@ -212,9 +212,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### strct.b _c1 -four 4501 six 4501 two 4501 +four 4501 PREHOOK: query: SELECT strct, mp, lst, str, count(val) FROM orc_create_complex GROUP BY strct, mp, lst, str PREHOOK: type: QUERY PREHOOK: Input: default@orc_create_complex @@ -224,6 +224,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### strct mp lst str _c4 -{"a":"one","b":"two"} {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] line1 4501 {"a":"three","b":"four"} {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] line2 4501 +{"a":"one","b":"two"} {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] line1 4501 {"a":"five","b":"six"} {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] line3 4501 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out index 867126f..c271b82 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_udf.q.out @@ -2368,23 +2368,23 @@ POSTHOOK: query: SELECT value, stddev(key), variance(key) FROM DECIMAL_UDF GROUP POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_udf #### A masked pattern was here #### +4 0.0 0.0 -1234567890 0.0 0.0 --1255 0.0 0.0 --11 0.0 0.0 --1 0.0 0.0 0 0.22561046704494161 0.050900082840236685 1 0.05928102563215321 0.0035142400000000066 2 0.0 0.0 3 0.0 0.0 -4 0.0 0.0 -10 0.0 0.0 -20 0.0 0.0 -100 0.0 0.0 124 0.0 0.0 -125 0.0 0.0 200 0.0 0.0 4400 0.0 0.0 1234567890 0.0 0.0 +10 0.0 0.0 +125 0.0 0.0 +-1255 0.0 0.0 +-11 0.0 0.0 +-1 0.0 0.0 +20 0.0 0.0 +100 0.0 0.0 PREHOOK: query: EXPLAIN SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF GROUP BY value @@ -2455,23 +2455,23 @@ POSTHOOK: query: SELECT value, stddev_samp(key), var_samp(key) FROM DECIMAL_UDF POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_udf #### A masked pattern was here #### +4 0.0 0.0 -1234567890 0.0 0.0 --1255 0.0 0.0 --11 0.0 0.0 --1 0.0 0.0 0 0.2348228191855647 0.055141756410256405 1 0.06627820154470102 0.004392800000000008 2 0.0 0.0 3 0.0 0.0 -4 0.0 0.0 -10 0.0 0.0 -20 0.0 0.0 -100 0.0 0.0 124 0.0 0.0 -125 0.0 0.0 200 0.0 0.0 4400 0.0 0.0 1234567890 0.0 0.0 +10 0.0 0.0 +125 0.0 0.0 +-1255 0.0 0.0 +-11 0.0 0.0 +-1 0.0 0.0 +20 0.0 0.0 +100 0.0 0.0 PREHOOK: query: EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT histogram_numeric(key, 3) FROM DECIMAL_UDF http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out index 91d9e88..f731cee 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_limit.q.out @@ -29,10 +29,10 @@ POSTHOOK: Lineage: t1.b SIMPLE [(t1_text)t1_text.FieldSchema(name:b, type:string POSTHOOK: Lineage: t1.c SIMPLE [(t1_text)t1_text.FieldSchema(name:c, type:string, comment:null), ] t1_text.a t1_text.b t1_text.c PREHOOK: query: EXPLAIN -SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10 +SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10 +SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -45,6 +45,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -85,16 +86,29 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -102,11 +116,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10 +PREHOOK: query: SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a, b, count(*) from T1 group by a, b with cube LIMIT 10 +POSTHOOK: query: SELECT a, b, count(*) from T1 group by a, b with cube order by a, b LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### @@ -122,10 +136,10 @@ NULL 2 3 NULL 3 1 NULL NULL 6 PREHOOK: query: EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10 +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10 +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -138,6 +152,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -178,16 +193,29 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 12 Data size: 3060 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 10 Data size: 2550 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -195,11 +223,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10 +PREHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) LIMIT 10 +POSTHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b), b, ()) order by a, b LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### @@ -215,10 +243,10 @@ NULL 2 3 NULL 3 1 NULL NULL 6 PREHOOK: query: EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10 +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10 +SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -231,6 +259,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -271,16 +300,29 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string), _col3 (type: bigint) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TopN Hash Memory Usage: 0.1 + value expressions: _col2 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 6 Data size: 1530 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -288,11 +330,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10 +PREHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) LIMIT 10 +POSTHOOK: query: SELECT a, b, count(*) FROM T1 GROUP BY a, b GROUPING SETS (a, (a, b)) order by a, b LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### @@ -308,10 +350,10 @@ a b _c2 5 NULL 1 8 NULL 1 PREHOOK: query: EXPLAIN -SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10 +SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10 +SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -324,6 +366,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -361,16 +404,28 @@ STAGE PLANS: expressions: _col0 (type: string) outputColumnNames: _col0 Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 2295 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -378,11 +433,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10 +PREHOOK: query: SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) LIMIT 10 +POSTHOOK: query: SELECT a FROM T1 GROUP BY a, b, c GROUPING SETS (a, b, c) order by a LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### @@ -398,10 +453,10 @@ NULL NULL NULL PREHOOK: query: EXPLAIN -SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10 +SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10 +SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -414,6 +469,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -447,6 +503,18 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE pruneGroupingSetId: true + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 10 Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE @@ -464,11 +532,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10 +PREHOOK: query: SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) LIMIT 10 +POSTHOOK: query: SELECT a FROM T1 GROUP BY a GROUPING SETS ((a), (a)) order by a LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### @@ -479,10 +547,10 @@ a 5 8 PREHOOK: query: EXPLAIN -SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10 +SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN -SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10 +SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10 POSTHOOK: type: QUERY Explain STAGE DEPENDENCIES: @@ -495,6 +563,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -535,16 +604,29 @@ STAGE PLANS: expressions: _col0 (type: double), _col2 (type: bigint) outputColumnNames: _col0, _col1 Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 10 + Reduce Output Operator + key expressions: _col0 (type: double) + sort order: + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TopN Hash Memory Usage: 0.1 + value expressions: _col1 (type: bigint) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 10 + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 765 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -552,15 +634,15 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10 +PREHOOK: query: SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@t1 #### A masked pattern was here #### -POSTHOOK: query: SELECT a + b, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) LIMIT 10 +POSTHOOK: query: SELECT a + b ab, count(*) FROM T1 GROUP BY a + b GROUPING SETS (a+b) order by ab LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 #### A masked pattern was here #### -_c0 _c1 +ab _c1 2.0 1 4.0 1 5.0 2 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out index 02fc90d..0ba6195 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_reduce.q.out @@ -217,6 +217,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain vectorization expression @@ -225,6 +226,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 POSTHOOK: type: QUERY PLAN VECTORIZATION: @@ -241,6 +243,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -312,6 +315,34 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: No TopN IS false + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Reducer 3 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + groupByVectorOutput: true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0] + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 Limit Vectorization: @@ -340,6 +371,7 @@ PREHOOK: query: select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@store_sales @@ -349,6 +381,7 @@ POSTHOOK: query: select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@store_sales http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out index 8849db7..9d51982 100644 --- a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out @@ -247,12 +247,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@store #### A masked pattern was here #### NULL -AAAAAAAABAAAAAAA -AAAAAAAACAAAAAAA AAAAAAAAEAAAAAAA AAAAAAAAHAAAAAAA AAAAAAAAIAAAAAAA AAAAAAAAKAAAAAAA +AAAAAAAABAAAAAAA +AAAAAAAACAAAAAAA PREHOOK: query: explain vectorization expression select s_store_id, GROUPING__ID from store @@ -384,12 +384,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@store #### A masked pattern was here #### NULL 1 -AAAAAAAABAAAAAAA 0 -AAAAAAAACAAAAAAA 0 AAAAAAAAEAAAAAAA 0 AAAAAAAAHAAAAAAA 0 AAAAAAAAIAAAAAAA 0 AAAAAAAAKAAAAAAA 0 +AAAAAAAABAAAAAAA 0 +AAAAAAAACAAAAAAA 0 PREHOOK: query: explain select s_store_id, GROUPING__ID from store http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out index 77abb3b..2870d9e 100644 --- a/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out @@ -234,15 +234,15 @@ POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### 2010-10-20 11 2010-10-21 12 -2010-10-22 11 +2010-10-26 13 +2010-10-30 11 2010-10-23 12 2010-10-24 12 2010-10-25 12 -2010-10-26 13 -2010-10-27 11 +2010-10-22 11 2010-10-28 12 2010-10-29 12 -2010-10-30 11 +2010-10-27 11 2010-10-31 8 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc sort by fl_num, fl_date limit 25 @@ -534,15 +534,15 @@ POSTHOOK: Input: default@flights_tiny_orc #### A masked pattern was here #### 2010-10-20 11 2010-10-21 12 -2010-10-22 11 +2010-10-26 13 +2010-10-30 11 2010-10-23 12 2010-10-24 12 2010-10-25 12 -2010-10-26 13 -2010-10-27 11 +2010-10-22 11 2010-10-28 12 2010-10-29 12 -2010-10-30 11 +2010-10-27 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_date ( origin_city_name STRING, @@ -913,15 +913,15 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### 2010-10-20 11 2010-10-21 12 -2010-10-22 11 +2010-10-26 13 +2010-10-30 11 2010-10-23 12 2010-10-24 12 2010-10-25 12 -2010-10-26 13 -2010-10-27 11 +2010-10-22 11 2010-10-28 12 2010-10-29 12 -2010-10-30 11 +2010-10-27 11 2010-10-31 8 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc_partitioned_date @@ -1492,15 +1492,15 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_date@fl_date=2010-10-31 #### A masked pattern was here #### 2010-10-20 11 2010-10-21 12 -2010-10-22 11 +2010-10-26 13 +2010-10-30 11 2010-10-23 12 2010-10-24 12 2010-10-25 12 -2010-10-26 13 -2010-10-27 11 +2010-10-22 11 2010-10-28 12 2010-10-29 12 -2010-10-30 11 +2010-10-27 11 2010-10-31 8 PREHOOK: query: CREATE TABLE flights_tiny_orc_partitioned_timestamp ( origin_city_name STRING, @@ -1870,17 +1870,17 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### 2010-10-20 07:00:00 11 -2010-10-21 07:00:00 12 -2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 +2010-10-22 07:00:00 11 2010-10-26 07:00:00 13 +2010-10-29 07:00:00 12 +2010-10-31 07:00:00 8 +2010-10-21 07:00:00 12 +2010-10-24 07:00:00 12 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 -2010-10-29 07:00:00 12 2010-10-30 07:00:00 11 -2010-10-31 07:00:00 8 PREHOOK: query: explain vectorization expression select * from flights_tiny_orc_partitioned_timestamp PREHOOK: type: QUERY @@ -2449,14 +2449,14 @@ POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10- POSTHOOK: Input: default@flights_tiny_orc_partitioned_timestamp@fl_time=2010-10-31 07%3A00%3A00 #### A masked pattern was here #### 2010-10-20 07:00:00 11 -2010-10-21 07:00:00 12 -2010-10-22 07:00:00 11 2010-10-23 07:00:00 12 -2010-10-24 07:00:00 12 2010-10-25 07:00:00 12 +2010-10-22 07:00:00 11 2010-10-26 07:00:00 13 +2010-10-29 07:00:00 12 +2010-10-31 07:00:00 8 +2010-10-21 07:00:00 12 +2010-10-24 07:00:00 12 2010-10-27 07:00:00 11 2010-10-28 07:00:00 12 -2010-10-29 07:00:00 12 2010-10-30 07:00:00 11 -2010-10-31 07:00:00 8 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out index 6000ec4..cf2db94 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out @@ -216,4 +216,4 @@ POSTHOOK: query: select sum(distinct cint), count(distinct cint), avg(distinct c POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### --3482841611 6082 -572647.4204209142 6.153814687328991E8 +-3482841611 6082 -572647.4204209142 6.153814687328982E8 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out index 10fc0f3..5a7a101 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_semijoin_reduction.q.out @@ -95,7 +95,7 @@ STAGE PLANS: TableScan alias: b filterExpr: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -105,7 +105,7 @@ STAGE PLANS: native: true predicateExpression: SelectColumnIsNotNull(col 1) -> boolean predicate: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_int (type: int) outputColumnNames: _col0 @@ -113,7 +113,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -122,7 +122,7 @@ STAGE PLANS: className: VectorReduceSinkLongOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 @@ -130,7 +130,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -332,7 +332,7 @@ STAGE PLANS: TableScan alias: b filterExpr: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -342,7 +342,7 @@ STAGE PLANS: native: true predicateExpression: SelectColumnIsNotNull(col 0) -> boolean predicate: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_str (type: string) outputColumnNames: _col0 @@ -350,7 +350,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -359,7 +359,7 @@ STAGE PLANS: className: VectorReduceSinkStringOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -367,7 +367,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -569,7 +569,7 @@ STAGE PLANS: TableScan alias: b filterExpr: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -579,7 +579,7 @@ STAGE PLANS: native: true predicateExpression: SelectColumnIsNotNull(col 0) -> boolean predicate: key_str is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_str (type: string) outputColumnNames: _col0 @@ -587,7 +587,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + @@ -596,7 +596,7 @@ STAGE PLANS: className: VectorReduceSinkStringOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -604,7 +604,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -807,7 +807,7 @@ STAGE PLANS: TableScan alias: b filterExpr: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -817,7 +817,7 @@ STAGE PLANS: native: true predicateExpression: SelectColumnIsNotNull(col 1) -> boolean predicate: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_int (type: int) outputColumnNames: _col0 @@ -825,7 +825,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -834,7 +834,7 @@ STAGE PLANS: className: VectorReduceSinkLongOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 @@ -842,7 +842,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -877,7 +877,7 @@ STAGE PLANS: TableScan alias: c filterExpr: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -887,7 +887,7 @@ STAGE PLANS: native: true predicateExpression: SelectColumnIsNotNull(col 1) -> boolean predicate: key_int is not null (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_int (type: int) outputColumnNames: _col0 @@ -895,7 +895,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -904,7 +904,7 @@ STAGE PLANS: className: VectorReduceSinkLongOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 @@ -912,7 +912,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -1146,7 +1146,7 @@ STAGE PLANS: TableScan alias: b filterExpr: (key_str is not null and key_int is not null) (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -1156,7 +1156,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 0) -> boolean, SelectColumnIsNotNull(col 1) -> boolean) -> boolean predicate: (key_str is not null and key_int is not null) (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_str (type: string), key_int (type: int) outputColumnNames: _col0, _col1 @@ -1164,7 +1164,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0, 1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: int) sort order: ++ @@ -1173,7 +1173,7 @@ STAGE PLANS: className: VectorReduceSinkMultiKeyOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 @@ -1181,7 +1181,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [0] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -1208,7 +1208,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=114) Group By Vectorization: @@ -1439,7 +1439,7 @@ STAGE PLANS: TableScan alias: b filterExpr: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) - Statistics: Num rows: 57 Data size: 10146 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 57 Data size: 10182 Basic stats: COMPLETE Column stats: NONE TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2] @@ -1449,7 +1449,7 @@ STAGE PLANS: native: true predicateExpression: FilterExprAndExpr(children: FilterStringColumnInList(col 2, values nonexistent1, nonexistent2) -> boolean, SelectColumnIsNotNull(col 1) -> boolean) -> boolean predicate: ((value) IN ('nonexistent1', 'nonexistent2') and key_int is not null) (type: boolean) - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5180 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key_int (type: int) outputColumnNames: _col0 @@ -1457,7 +1457,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5180 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + @@ -1466,7 +1466,7 @@ STAGE PLANS: className: VectorReduceSinkLongOperator native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5180 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int) outputColumnNames: _col0 @@ -1474,7 +1474,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumns: [1] - Statistics: Num rows: 29 Data size: 5162 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 29 Data size: 5180 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=58) Group By Vectorization: http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out index 147f5af..13eae75 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_parquet.q.out @@ -218,131 +218,131 @@ POSTHOOK: query: select ctinyint, POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypes_parquet #### A masked pattern was here #### -NULL 1073418988 -16379 3115 NULL 305051.4870777435 --64 626923679 -15920 21 -64.0 9254.456539277186 --63 626923679 -12516 16 -63.0 9263.605837223322 --62 626923679 -15992 24 -62.0 9004.593091474135 --61 626923679 -15142 22 -61.0 9357.236187870849 --60 626923679 -15792 24 -60.0 9892.656196775464 -59 626923679 -15789 28 -59.0 9829.790704244733 -58 626923679 -15169 20 -58.0 9549.096672008198 --57 626923679 -14893 32 -57.0 8572.083461570477 --56 626923679 -11999 33 -56.0 9490.842152672341 --55 626923679 -13381 26 -55.0 9157.562103946742 -54 626923679 -14815 23 -54.0 9614.154026896626 --53 626923679 -15445 19 -53.0 9387.739325499799 --52 626923679 -16369 30 -52.0 8625.06871423408 --51 1073680599 -15734 1028 -51.0 9531.569305177045 -50 626923679 -14320 27 -50.0 8548.827748002343 --49 626923679 -14831 23 -49.0 9894.429191738676 --48 626923679 -15462 26 -48.0 9913.883371354861 --47 626923679 -16096 19 -47.0 9011.009178780589 --46 626923679 -12427 21 -46.0 9182.943188188632 --45 626923679 -15027 21 -45.0 8567.489593562543 --44 626923679 -15667 21 -44.0 10334.01810499552 --43 626923679 -15607 27 -43.0 8715.255026265124 --42 626923679 -16025 14 -42.0 9692.646755759979 -41 626923679 -12606 21 -41.0 9034.40949481481 --40 626923679 -14678 23 -40.0 9883.334986561835 --39 626923679 -15612 19 -39.0 9765.551806305297 -38 626923679 -14914 28 -38.0 8767.375358291503 -37 626923679 -14780 17 -37.0 10368.905538788269 -36 626923679 -16208 23 -36.0 8773.547684436919 --35 626923679 -16059 23 -35.0 10136.580492864763 -34 626923679 -15450 29 -34.0 8708.243526705026 --33 626923679 -12779 21 -33.0 8854.331159704514 --32 626923679 -15866 25 -32.0 9535.546396775915 --31 626923679 -15915 22 -31.0 9187.596784112568 --30 626923679 -14863 23 -30.0 9193.941914019653 --29 626923679 -14747 26 -29.0 9052.945656011721 --28 626923679 -15813 20 -28.0 9616.869413270924 --27 626923679 -14984 20 -27.0 8465.29660255097 -26 626923679 -15686 15 -26.0 10874.523900405318 --25 626923679 -15862 24 -25.0 9778.256724727018 -24 626923679 -16311 26 -24.0 9386.736402961187 --23 626923679 -16355 36 -23.345263230173213 9401.831290253447 -22 626923679 -14701 22 -22.0 8809.230165774987 +-18 626923679 -14863 24 -18.0 9638.430684071413 +-12 626923679 -16373 22 -12.0 10173.15707541171 +-10 626923679 -15384 28 -10.0 8850.451610567823 +-9 626923679 -15329 31 -9.0 8999.391457373968 +-2 626923679 -16277 20 -2.0 10800.090249507177 +14 626923679 -13367 28 14.0 9074.674998750581 +17 626923679 -16109 22 16.73235294865627 1353416.3383574807 +19 626923679 -16049 21 19.0 9423.560227007669 +20 626923679 -15149 21 20.0 11161.893298093504 +21 626923679 -15931 23 21.0 9683.044864861204 +26 626923679 -14516 29 26.0 9123.125508880432 +27 626923679 -14965 24 27.0 9802.871860196345 +30 626923679 -14111 27 30.0 10066.520234676527 +34 626923679 -15059 28 34.0 8756.731536033676 +36 626923679 -15912 20 36.0 9475.257975138164 +38 626923679 -15248 29 38.0 9900.256257785535 +40 626923679 -15861 22 40.0 9283.318678549174 +44 626923679 -11185 16 44.0 9425.076634933797 +45 626923679 -15228 18 45.0 9459.968668643689 +49 626923679 -15923 27 49.0 9850.111848934683 +53 626923679 -16217 30 53.0 9895.247408969733 +58 626923679 -13627 20 58.0 9083.529665947459 +NULL 1073418988 -16379 3115 NULL 305051.4870777435 +-60 626923679 -15792 24 -60.0 9892.656196775464 +-56 626923679 -11999 33 -56.0 9490.842152672341 +-49 626923679 -14831 23 -49.0 9894.429191738676 +-46 626923679 -12427 21 -46.0 9182.943188188632 +-45 626923679 -15027 21 -45.0 8567.489593562543 +-42 626923679 -16025 14 -42.0 9692.646755759979 +-32 626923679 -15866 25 -32.0 9535.546396775915 +-28 626923679 -15813 20 -28.0 9616.869413270924 +-27 626923679 -14984 20 -27.0 8465.29660255097 -21 626923679 -16017 27 -21.0 9480.349236669877 -20 626923679 -16126 24 -20.0 9868.92268080106 -19 626923679 -15935 25 -19.0 9967.22240685782 --18 626923679 -14863 24 -18.0 9638.430684071413 -17 626923679 -15922 19 -17.0 9944.104273894172 --16 626923679 -15154 21 -16.0 8884.207393686478 --15 626923679 -16036 24 -15.0 9450.506254395024 -14 626923679 -13884 22 -14.0 10125.818731386042 --13 626923679 -15446 30 -13.0 8907.942987576693 --12 626923679 -16373 22 -12.0 10173.15707541171 --11 626923679 -15659 32 -11.0 10453.738567408038 --10 626923679 -15384 28 -10.0 8850.451610567823 --9 626923679 -15329 31 -9.0 8999.391457373968 -8 626923679 -14678 18 -8.0 9976.831992670684 --7 626923679 -14584 23 -7.0 9946.605446407746 -6 626923679 -15980 30 -6.0 10262.829252317424 +0 626923679 -14254 24 0.0 10057.5018088718 +9 626923679 -13629 25 9.0 10157.217948808622 +10 626923679 -15887 26 10.0 9104.820520135108 +28 626923679 -14455 20 28.0 9283.289383115296 +37 626923679 -12081 24 37.0 9017.860034890362 +41 626923679 -13480 21 41.0 9016.291129937847 +48 626923679 -16372 29 48.0 10079.286173063345 +52 626923679 -15450 20 52.0 9261.723648435052 +59 626923679 -16076 17 59.0 10117.44967077967 +-64 626923679 -15920 21 -64.0 9254.456539277186 +-63 626923679 -12516 16 -63.0 9263.605837223322 +-62 626923679 -15992 24 -62.0 9004.593091474135 +-61 626923679 -15142 22 -61.0 9357.236187870849 +-55 626923679 -13381 26 -55.0 9157.562103946742 +-53 626923679 -15445 19 -53.0 9387.739325499799 +-52 626923679 -16369 30 -52.0 8625.06871423408 +-51 1073680599 -15734 1028 -51.0 9531.569305177045 +-29 626923679 -14747 26 -29.0 9052.945656011721 +-15 626923679 -16036 24 -15.0 9450.506254395024 +-13 626923679 -15446 30 -13.0 8907.942987576693 +-7 626923679 -14584 23 -7.0 9946.605446407746 -5 626923679 -15780 24 -5.0 10599.227726422314 -4 626923679 -16207 21 -4.0 9682.726604102581 -3 626923679 -13632 16 -3.0 8836.215573422822 --2 626923679 -16277 20 -2.0 10800.090249507177 -1 626923679 -15441 36 -1.0486250072717667 8786.246963933321 -0 626923679 -14254 24 0.0 10057.5018088718 +4 626923679 -15999 29 4.0 9516.189702058042 +6 626923679 -15948 30 6.0 9644.247255286113 +16 626923679 -14001 26 16.0 10130.883606275334 +18 626923679 -15779 21 18.0 10820.004053788869 +31 626923679 -15960 24 31.0 10427.970184550613 +33 626923679 -14642 29 40.61776386607777 1304429.5939037625 +39 626923679 -14887 28 39.0 10513.343644635232 +43 626923679 -15703 28 43.0 8757.796089055722 +46 626923679 -15187 22 46.0 9685.908173160062 +47 626923679 -16324 22 47.0 9822.220821743611 +50 626923679 -16236 21 50.0 9398.176197406601 +56 626923679 -12631 21 56.0 8860.917133763547 +57 626923679 -15620 25 57.0 9413.99393840875 +60 626923679 -13606 23 60.0 8346.267436552042 +-57 626923679 -14893 32 -57.0 8572.083461570477 +-48 626923679 -15462 26 -48.0 9913.883371354861 +-47 626923679 -16096 19 -47.0 9011.009178780589 +-44 626923679 -15667 21 -44.0 10334.01810499552 +-43 626923679 -15607 27 -43.0 8715.255026265124 +-40 626923679 -14678 23 -40.0 9883.334986561835 +-39 626923679 -15612 19 -39.0 9765.551806305297 +-35 626923679 -16059 23 -35.0 10136.580492864763 +-33 626923679 -12779 21 -33.0 8854.331159704514 +-31 626923679 -15915 22 -31.0 9187.596784112568 +-30 626923679 -14863 23 -30.0 9193.941914019653 +-25 626923679 -15862 24 -25.0 9778.256724727018 +-23 626923679 -16355 36 -23.345263230173213 9401.831290253447 +-16 626923679 -15154 21 -16.0 8884.207393686478 +-11 626923679 -15659 32 -11.0 10453.738567408038 1 626923679 -14610 30 1.0 10016.486277900643 2 626923679 -16227 25 2.0 10083.276127543355 3 626923679 -16339 30 3.0 10483.526375885149 -4 626923679 -15999 29 4.0 9516.189702058042 5 626923679 -16169 31 5.0 11114.001902469323 -6 626923679 -15948 30 6.0 9644.247255286113 7 626923679 -15839 25 7.0 10077.151640330823 8 1070764888 -15778 1034 8.0 9562.355155774725 -9 626923679 -13629 25 9.0 10157.217948808622 -10 626923679 -15887 26 10.0 9104.820520135108 11 1072654057 -14696 1035 11.0 9531.018991371746 12 626923679 -14642 18 12.0 9696.038286378725 13 626923679 -14771 26 13.0 8128.265919972384 -14 626923679 -13367 28 14.0 9074.674998750581 15 626923679 -16339 28 15.0 9770.473400901916 -16 626923679 -14001 26 16.0 10130.883606275334 -17 626923679 -16109 22 16.73235294865627 1353416.3383574807 -18 626923679 -15779 21 18.0 10820.004053788869 -19 626923679 -16049 21 19.0 9423.560227007669 -20 626923679 -15149 21 20.0 11161.893298093504 -21 626923679 -15931 23 21.0 9683.044864861204 22 626923679 -16280 26 22.0 9693.155720861765 23 626923679 -15514 24 23.0 8542.419116415425 24 626923679 -15086 24 24.0 9661.203790645088 25 626923679 -11349 23 25.0 8888.959012093468 -26 626923679 -14516 29 26.0 9123.125508880432 -27 626923679 -14965 24 27.0 9802.871860196345 -28 626923679 -14455 20 28.0 9283.289383115296 29 626923679 -15892 16 29.0 9874.046501817154 -30 626923679 -14111 27 30.0 10066.520234676527 -31 626923679 -15960 24 31.0 10427.970184550613 32 626923679 -14044 24 32.0 8376.464579403413 -33 626923679 -14642 29 40.61776386607777 1304429.5939037625 -34 626923679 -15059 28 34.0 8756.731536033676 35 626923679 -16153 27 35.0 10351.008404963042 -36 626923679 -15912 20 36.0 9475.257975138164 -37 626923679 -12081 24 37.0 9017.860034890362 -38 626923679 -15248 29 38.0 9900.256257785535 -39 626923679 -14887 28 39.0 10513.343644635232 -40 626923679 -15861 22 40.0 9283.318678549174 -41 626923679 -13480 21 41.0 9016.291129937847 42 626923679 -15834 28 42.0 10318.01399719996 -43 626923679 -15703 28 43.0 8757.796089055722 -44 626923679 -11185 16 44.0 9425.076634933797 -45 626923679 -15228 18 45.0 9459.968668643689 -46 626923679 -15187 22 46.0 9685.908173160062 -47 626923679 -16324 22 47.0 9822.220821743611 -48 626923679 -16372 29 48.0 10079.286173063345 -49 626923679 -15923 27 49.0 9850.111848934683 -50 626923679 -16236 21 50.0 9398.176197406601 51 626923679 -15790 17 51.0 9220.075799194028 -52 626923679 -15450 20 52.0 9261.723648435052 -53 626923679 -16217 30 53.0 9895.247408969733 54 626923679 -15245 16 54.0 9789.50878424882 55 626923679 -15887 21 55.0 9826.38569192808 -56 626923679 -12631 21 56.0 8860.917133763547 -57 626923679 -15620 25 57.0 9413.99393840875 -58 626923679 -13627 20 58.0 9083.529665947459 -59 626923679 -16076 17 59.0 10117.44967077967 -60 626923679 -13606 23 60.0 8346.267436552042 61 626923679 -15894 29 61.0 8785.714950987198 62 626923679 -14307 17 62.0 9491.752726667326 http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out b/ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out index 371ebb2..6422bd5 100644 --- a/ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out +++ b/ql/src/test/results/clientpositive/vector_binary_join_groupby.q.out @@ -96,24 +96,27 @@ POSTHOOK: Lineage: hundredorc.si SIMPLE [(over1k)over1k.FieldSchema(name:si, typ POSTHOOK: Lineage: hundredorc.t SIMPLE [(over1k)over1k.FieldSchema(name:t, type:tinyint, comment:null), ] POSTHOOK: Lineage: hundredorc.ts SIMPLE [(over1k)over1k.FieldSchema(name:ts, type:timestamp, comment:null), ] PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION -SELECT sum(hash(*)) +SELECT sum(hash(*)) k FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION -SELECT sum(hash(*)) +SELECT sum(hash(*)) k FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k POSTHOOK: type: QUERY PLAN VECTORIZATION: enabled: true enabledConditionsMet: [hive.vectorized.execution.enabled IS true] STAGE DEPENDENCIES: - Stage-5 is a root stage - Stage-2 depends on stages: Stage-5 - Stage-0 depends on stages: Stage-2 + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-3 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-3 STAGE PLANS: - Stage: Stage-5 + Stage: Stage-6 Map Reduce Local Work Alias -> Map Local Tables: $hdt$_0:$hdt$_0:t1 @@ -229,6 +232,47 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + TableScan Vectorization: + native: true + projectedOutputColumns: [0] + Reduce Output Operator + key expressions: _col0 (type: bigint) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat @@ -241,13 +285,15 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: SELECT sum(hash(*)) +PREHOOK: query: SELECT sum(hash(*)) k FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k PREHOOK: type: QUERY PREHOOK: Input: default@hundredorc #### A masked pattern was here #### -POSTHOOK: query: SELECT sum(hash(*)) +POSTHOOK: query: SELECT sum(hash(*)) k FROM hundredorc t1 JOIN hundredorc t2 ON t1.bin = t2.bin +order by k POSTHOOK: type: QUERY POSTHOOK: Input: default@hundredorc #### A masked pattern was here #### @@ -256,11 +302,13 @@ PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), bin FROM hundredorc GROUP BY bin +order by bin PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT count(*), bin FROM hundredorc GROUP BY bin +order by bin POSTHOOK: type: QUERY PLAN VECTORIZATION: enabled: true @@ -268,7 +316,8 @@ PLAN VECTORIZATION: STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -342,11 +391,53 @@ STAGE PLANS: Statistics: Num rows: 50 Data size: 14819 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 50 Data size: 14819 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1] + Reduce Output Operator + key expressions: _col1 (type: binary) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 50 Data size: 14819 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: bigint), KEY.reducesinkkey0 (type: binary) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 50 Data size: 14819 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 14819 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -357,12 +448,14 @@ STAGE PLANS: PREHOOK: query: SELECT count(*), bin FROM hundredorc GROUP BY bin +order by bin PREHOOK: type: QUERY PREHOOK: Input: default@hundredorc #### A masked pattern was here #### POSTHOOK: query: SELECT count(*), bin FROM hundredorc GROUP BY bin +order by bin POSTHOOK: type: QUERY POSTHOOK: Input: default@hundredorc #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/10bd23eb/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out b/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out index e0e11ed..43ce596 100644 --- a/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out +++ b/ql/src/test/results/clientpositive/vector_groupby_reduce.q.out @@ -217,6 +217,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain vectorization expression @@ -225,6 +226,7 @@ select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 POSTHOOK: type: QUERY PLAN VECTORIZATION: @@ -233,7 +235,8 @@ PLAN VECTORIZATION: STAGE DEPENDENCIES: Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 STAGE PLANS: Stage: Stage-1 @@ -298,6 +301,48 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + TableScan Vectorization: + native: true + projectedOutputColumns: [0] + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false, No TopN IS false + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + groupByVectorOutput: true + inputFileFormats: org.apache.hadoop.mapred.SequenceFileInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 44138 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 20 Statistics: Num rows: 20 Data size: 1760 Basic stats: COMPLETE Column stats: NONE @@ -320,6 +365,7 @@ PREHOOK: query: select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@store_sales @@ -329,6 +375,7 @@ POSTHOOK: query: select from store_sales group by ss_ticket_number +order by ss_ticket_number limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@store_sales
