http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cross_prod_3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cross_prod_3.q.out b/ql/src/test/results/clientpositive/llap/cross_prod_3.q.out new file mode 100644 index 0000000..94fe942 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cross_prod_3.q.out @@ -0,0 +1,133 @@ +PREHOOK: query: create table X (key string, value string) clustered by (key) into 2 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@X +POSTHOOK: query: create table X (key string, value string) clustered by (key) into 2 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@X +PREHOOK: query: insert overwrite table X select distinct * from src order by key limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@x +POSTHOOK: query: insert overwrite table X select distinct * from src order by key limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@x +POSTHOOK: Lineage: x.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: x.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table Y as +select * from src order by key limit 1 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@Y +POSTHOOK: query: create table Y as +select * from src order by key limit 1 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@Y +POSTHOOK: Lineage: y.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: y.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select * from Y, (select * from X as A join X as B on A.key=B.key) as C where Y.key=C.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from Y, (select * from X as A join X as B on A.key=B.key) as C where Y.key=C.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 2 (CUSTOM_EDGE), Map 3 (CUSTOM_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + 2 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + input vertices: + 1 Map 2 + 2 Map 3 + Statistics: Num rows: 22 Data size: 8096 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col2 (type: string), _col3 (type: string), _col0 (type: string), _col1 (type: string), _col4 (type: string), _col5 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 22 Data size: 8096 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 22 Data size: 8096 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: llap + LLAP IO: no inputs + Map 2 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 368 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +
http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cross_prod_4.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cross_prod_4.q.out b/ql/src/test/results/clientpositive/llap/cross_prod_4.q.out new file mode 100644 index 0000000..999e018 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cross_prod_4.q.out @@ -0,0 +1,195 @@ +PREHOOK: query: create table X as +select distinct * from src order by key limit 10 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@X +POSTHOOK: query: create table X as +select distinct * from src order by key limit 10 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@X +POSTHOOK: Lineage: x.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: x.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: explain select * from X as A, X as B +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from X as A, X as B +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 3680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) + Execution mode: llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 100 Data size: 73700 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 73700 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[11][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: select * from X as A, X as B order by A.key, B.key +PREHOOK: type: QUERY +PREHOOK: Input: default@x +#### A masked pattern was here #### +POSTHOOK: query: select * from X as A, X as B order by A.key, B.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@x +#### A masked pattern was here #### +0 val_0 0 val_0 +0 val_0 10 val_10 +0 val_0 100 val_100 +0 val_0 103 val_103 +0 val_0 104 val_104 +0 val_0 105 val_105 +0 val_0 11 val_11 +0 val_0 111 val_111 +0 val_0 113 val_113 +0 val_0 114 val_114 +10 val_10 0 val_0 +10 val_10 10 val_10 +10 val_10 100 val_100 +10 val_10 103 val_103 +10 val_10 104 val_104 +10 val_10 105 val_105 +10 val_10 11 val_11 +10 val_10 111 val_111 +10 val_10 113 val_113 +10 val_10 114 val_114 +100 val_100 0 val_0 +100 val_100 10 val_10 +100 val_100 100 val_100 +100 val_100 103 val_103 +100 val_100 104 val_104 +100 val_100 105 val_105 +100 val_100 11 val_11 +100 val_100 111 val_111 +100 val_100 113 val_113 +100 val_100 114 val_114 +103 val_103 0 val_0 +103 val_103 10 val_10 +103 val_103 100 val_100 +103 val_103 103 val_103 +103 val_103 104 val_104 +103 val_103 105 val_105 +103 val_103 11 val_11 +103 val_103 111 val_111 +103 val_103 113 val_113 +103 val_103 114 val_114 +104 val_104 0 val_0 +104 val_104 10 val_10 +104 val_104 100 val_100 +104 val_104 103 val_103 +104 val_104 104 val_104 +104 val_104 105 val_105 +104 val_104 11 val_11 +104 val_104 111 val_111 +104 val_104 113 val_113 +104 val_104 114 val_114 +105 val_105 0 val_0 +105 val_105 10 val_10 +105 val_105 100 val_100 +105 val_105 103 val_103 +105 val_105 104 val_104 +105 val_105 105 val_105 +105 val_105 11 val_11 +105 val_105 111 val_111 +105 val_105 113 val_113 +105 val_105 114 val_114 +11 val_11 0 val_0 +11 val_11 10 val_10 +11 val_11 100 val_100 +11 val_11 103 val_103 +11 val_11 104 val_104 +11 val_11 105 val_105 +11 val_11 11 val_11 +11 val_11 111 val_111 +11 val_11 113 val_113 +11 val_11 114 val_114 +111 val_111 0 val_0 +111 val_111 10 val_10 +111 val_111 100 val_100 +111 val_111 103 val_103 +111 val_111 104 val_104 +111 val_111 105 val_105 +111 val_111 11 val_11 +111 val_111 111 val_111 +111 val_111 113 val_113 +111 val_111 114 val_114 +113 val_113 0 val_0 +113 val_113 10 val_10 +113 val_113 100 val_100 +113 val_113 103 val_103 +113 val_113 104 val_104 +113 val_113 105 val_105 +113 val_113 11 val_11 +113 val_113 111 val_111 +113 val_113 113 val_113 +113 val_113 114 val_114 +114 val_114 0 val_0 +114 val_114 10 val_10 +114 val_114 100 val_100 +114 val_114 103 val_103 +114 val_114 104 val_104 +114 val_114 105 val_105 +114 val_114 11 val_11 +114 val_114 111 val_111 +114 val_114 113 val_113 +114 val_114 114 val_114 http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out b/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out index 7dffba9..fcf53e4 100644 --- a/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out +++ b/ql/src/test/results/clientpositive/llap/cross_product_check_1.q.out @@ -42,7 +42,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -115,7 +115,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) - Reducer 3 <- Map 5 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 5 (XPROD_EDGE), Reducer 2 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -233,7 +233,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -364,8 +364,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 6 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) + Reducer 4 <- Map 3 (XPROD_EDGE), Map 6 (XPROD_EDGE) Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -488,7 +488,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 6 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 6 (XPROD_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) Reducer 6 <- Reducer 5 (SIMPLE_EDGE) #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out index 3b3d371..8a541b6 100644 --- a/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cross_product_check_2.q.out @@ -28,7 +28,7 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@B POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join B PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join B @@ -42,7 +42,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -54,26 +54,13 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3 - input vertices: - 1 Map 2 - Statistics: Num rows: 5000 Data size: 3596680 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5000 Data size: 3596680 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: b @@ -88,6 +75,24 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 5000 Data size: 3596680 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5000 Data size: 3596680 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -95,7 +100,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Map 3' is a cross product +Warning: Shuffle Join MERGEJOIN[21][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A PREHOOK: type: QUERY POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A @@ -109,8 +114,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) - Map 3 <- Map 1 (BROADCAST_EDGE) + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -133,7 +138,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0, _col1, _col2, _col3 input vertices: - 1 Map 2 + 1 Map 3 Statistics: Num rows: 11 Data size: 4048 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: @@ -141,7 +146,7 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: d2 @@ -161,7 +166,7 @@ STAGE PLANS: value expressions: _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: a @@ -170,25 +175,30 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - input vertices: - 0 Map 1 - Statistics: Num rows: 5500 Data size: 3956348 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 5500 Data size: 3956348 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 5500 Data size: 3956348 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 5500 Data size: 3956348 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -196,7 +206,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[26][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 on d1.key = d2.key @@ -216,9 +226,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Map 3 <- Map 5 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -230,26 +240,13 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - input vertices: - 1 Reducer 3 - Statistics: Num rows: 2500 Data size: 1338340 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 2500 Data size: 1338340 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: d1 @@ -269,7 +266,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 4 + 1 Map 5 Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -283,7 +280,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 4 + Map 5 Map Operator Tree: TableScan alias: d2 @@ -302,7 +299,25 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 2500 Data size: 1338340 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2500 Data size: 1338340 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -321,8 +336,8 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 2' is a cross product -Warning: Map Join MAPJOIN[20][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_1, $hdt$_2]] in Stage 'Reducer 4' is a cross product +Warning: Shuffle Join MERGEJOIN[20][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 PREHOOK: type: QUERY POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1 @@ -336,9 +351,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Reducer 3 (BROADCAST_EDGE) - Map 2 <- Map 4 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) + Reducer 4 <- Map 3 (XPROD_EDGE), Map 6 (XPROD_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -350,26 +365,13 @@ STAGE PLANS: expressions: key (type: string), value (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1, _col2 - input vertices: - 1 Reducer 3 - Statistics: Num rows: 25000 Data size: 13508400 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25000 Data size: 13508400 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 175168 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: d1 @@ -378,29 +380,13 @@ STAGE PLANS: expressions: key (type: string) outputColumnNames: _col0 Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - input vertices: - 1 Map 4 - Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE - Group By Operator - keys: _col0 (type: string) - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 10 Data size: 1840 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) Execution mode: llap LLAP IO: no inputs - Map 4 + Map 6 Map Operator Tree: TableScan alias: d2 @@ -412,7 +398,46 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 25000 Data size: 13508400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25000 Data size: 13508400 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: _col0 (type: string) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 100 Data size: 18900 Basic stats: COMPLETE Column stats: NONE + Reducer 5 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -431,7 +456,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[30][bigTable=?] in task 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[30][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 3' is a cross product PREHOOK: query: explain select * from (select A.key from A group by key) ss join (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1 @@ -449,9 +474,10 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 3 <- Map 5 (BROADCAST_EDGE) - Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (BROADCAST_EDGE) - Reducer 4 <- Map 3 (SIMPLE_EDGE) + Map 4 <- Map 6 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -475,7 +501,7 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 87584 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: d1 @@ -495,7 +521,7 @@ STAGE PLANS: 1 _col0 (type: string) outputColumnNames: _col0 input vertices: - 1 Map 5 + 1 Map 6 Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: _col0 (type: string) @@ -509,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 11 Data size: 2024 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Map 5 + Map 6 Map Operator Tree: TableScan alias: d2 @@ -536,24 +562,29 @@ STAGE PLANS: mode: mergepartial outputColumnNames: _col0 Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col1 - input vertices: - 1 Reducer 4 + Reduce Output Operator + sort order: + Statistics: Num rows: 250 Data size: 43792 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1250 Data size: 450210 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 1250 Data size: 450210 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1250 Data size: 450210 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Reducer 4 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 Execution mode: llap Reduce Operator Tree: Group By Operator http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cte_5.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cte_5.q.out b/ql/src/test/results/clientpositive/llap/cte_5.q.out index 88d43d0..dad6cd8 100644 --- a/ql/src/test/results/clientpositive/llap/cte_5.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_5.q.out @@ -77,7 +77,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) Stage-0 Fetch Operator @@ -89,15 +89,15 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_13] (rows=2 width=90) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Map 1 [XPROD_EDGE] llap + XPROD_EDGE [RS_6] Select Operator [SEL_2] (rows=1 width=4) Filter Operator [FIL_11] (rows=1 width=4) predicate:(UDFToDouble(colnum) = 5.0) TableScan [TS_0] (rows=1 width=4) mydb@q1,a,Tbl:COMPLETE,Col:NONE,Output:["colnum"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Map 3 [XPROD_EDGE] llap + XPROD_EDGE [RS_7] Select Operator [SEL_5] (rows=2 width=85) Filter Operator [FIL_12] (rows=2 width=87) predicate:(key = '5') http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out index a1156e8..9763698 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_1.q.out @@ -14,7 +14,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) Stage-0 Fetch Operator @@ -26,15 +26,15 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Map 1 [XPROD_EDGE] llap + XPROD_EDGE [RS_6] Select Operator [SEL_2] (rows=2 width=85) Filter Operator [FIL_11] (rows=2 width=87) predicate:(key = '5') TableScan [TS_0] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Map 3 [XPROD_EDGE] llap + XPROD_EDGE [RS_7] Select Operator [SEL_5] (rows=2 width=85) Filter Operator [FIL_12] (rows=2 width=87) predicate:(key = '5') http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out b/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out index a1156e8..9763698 100644 --- a/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out +++ b/ql/src/test/results/clientpositive/llap/cte_mat_2.q.out @@ -14,7 +14,7 @@ POSTHOOK: type: QUERY Plan optimized by CBO. Vertex dependency in root stage -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE) +Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) Stage-0 Fetch Operator @@ -26,15 +26,15 @@ Stage-0 Output:["_col0"] Merge Join Operator [MERGEJOIN_13] (rows=4 width=8) Conds:(Inner) - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_6] + <-Map 1 [XPROD_EDGE] llap + XPROD_EDGE [RS_6] Select Operator [SEL_2] (rows=2 width=85) Filter Operator [FIL_11] (rows=2 width=87) predicate:(key = '5') TableScan [TS_0] (rows=500 width=87) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"] - <-Map 3 [CUSTOM_SIMPLE_EDGE] llap - PARTITION_ONLY_SHUFFLE [RS_7] + <-Map 3 [XPROD_EDGE] llap + XPROD_EDGE [RS_7] Select Operator [SEL_5] (rows=2 width=85) Filter Operator [FIL_12] (rows=2 width=87) predicate:(key = '5') http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index de867b2..3c3cd74 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -2228,7 +2228,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### @@ -2351,7 +2351,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE) Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -4926,7 +4926,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 #### A masked pattern was here #### 1000 -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' @@ -4940,8 +4940,9 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 3 <- Map 1 (BROADCAST_EDGE), Map 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) + Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Map 1 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -4955,16 +4956,6 @@ STAGE PLANS: Reduce Output Operator sort order: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Execution mode: llap - LLAP IO: no inputs - Map 2 - Map Operator Tree: - TableScan - alias: srcpart - filterExpr: (ds = '2008-04-08') (type: boolean) - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: '2008-04-08' (type: string) mode: hash @@ -4977,35 +4968,26 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Reducer 2 Execution mode: llap Reduce Operator Tree: - Group By Operator - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - input vertices: - 0 Map 1 - Statistics: Num rows: 500000 Data size: 11124000 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: count() - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: bigint) - Reducer 4 + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + Statistics: Num rows: 500000 Data size: 11124000 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Reducer 3 Execution mode: llap Reduce Operator Tree: Group By Operator @@ -5020,6 +5002,19 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Stage: Stage-0 Fetch Operator @@ -5027,7 +5022,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[22][bigTable=?] in task 'Reducer 3' is a cross product +Warning: Shuffle Join MERGEJOIN[22][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from srcpart join (select ds as ds, ds as `date` from srcpart group by ds) s on (srcpart.ds = s.ds) where s.`date` = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out index c2d4edf..ff71fb2 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning_2.q.out @@ -566,7 +566,7 @@ bar baz baz baz -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT agg.amount FROM agg_01 agg, dim_shops d1 @@ -588,7 +588,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -607,7 +607,7 @@ STAGE PLANS: value expressions: _col0 (type: decimal(10,0)) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: d1 @@ -618,25 +618,29 @@ STAGE PLANS: Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Select Operator Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0 - input vertices: - 0 Map 1 - Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0 + Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 9 Data size: 1080 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -644,12 +648,12 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 2' is a cross product +Warning: Shuffle Join MERGEJOIN[15][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 WHERE agg.dim_shops_id = d1.id -and agg.dim_shops_id = 1 +and agg.dim_shops_id = 1 order by agg.amount PREHOOK: type: QUERY PREHOOK: Input: default@agg_01 PREHOOK: Input: default@agg_01@dim_shops_id=1 @@ -659,15 +663,15 @@ POSTHOOK: query: SELECT agg.amount FROM agg_01 agg, dim_shops d1 WHERE agg.dim_shops_id = d1.id -and agg.dim_shops_id = 1 +and agg.dim_shops_id = 1 order by agg.amount POSTHOOK: type: QUERY POSTHOOK: Input: default@agg_01 POSTHOOK: Input: default@agg_01@dim_shops_id=1 POSTHOOK: Input: default@dim_shops #### A masked pattern was here #### 1 -3 2 +3 PREHOOK: query: EXPLAIN SELECT d1.label, count(*), sum(agg.amount) FROM agg_01 agg, dim_shops d1 http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out index d7daa55..1c672ef 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_sw.q.out @@ -200,7 +200,7 @@ STAGE PLANS: Map 13 <- Reducer 12 (BROADCAST_EDGE), Reducer 9 (BROADCAST_EDGE) Reducer 12 <- Map 11 (CUSTOM_SIMPLE_EDGE) Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE), Reducer 8 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (XPROD_EDGE), Reducer 8 (XPROD_EDGE) Reducer 4 <- Reducer 3 (CUSTOM_SIMPLE_EDGE) Reducer 5 <- Map 1 (CUSTOM_SIMPLE_EDGE) Reducer 7 <- Map 6 (CUSTOM_SIMPLE_EDGE) http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/explainuser_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out index 4452dea..6e55acf 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out @@ -208,7 +208,7 @@ POSTHOOK: query: drop table src_orc_merge_test_part POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@src_orc_merge_test_part POSTHOOK: Output: default@src_orc_merge_test_part -Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[18][tables = [src1, src2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select sum(hash(a.k1,a.v1,a.k2, a.v2)) from ( select src1.key as k1, src1.value as v1, @@ -232,37 +232,39 @@ POSTHOOK: type: QUERY Plan not optimized by CBO. Vertex dependency in root stage -Map 1 <- Map 3 (BROADCAST_EDGE) -Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE) +Reducer 3 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) Stage-0 Fetch Operator limit:-1 Stage-1 - Reducer 2 llap + Reducer 3 llap File Output Operator [FS_15] Group By Operator [GBY_13] (rows=1 width=8) Output:["_col0"],aggregations:["sum(VALUE._col0)"] - <-Map 1 [CUSTOM_SIMPLE_EDGE] llap + <-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_12] Group By Operator [GBY_11] (rows=1 width=8) Output:["_col0"],aggregations:["sum(hash(_col0,_col1,_col2,_col3))"] - Map Join Operator [MAPJOIN_18] (rows=27556 width=356) + Merge Join Operator [MERGEJOIN_18] (rows=27556 width=356) Conds:(Inner),Output:["_col0","_col1","_col2","_col3"] - <-Map 3 [BROADCAST_EDGE] llap - BROADCAST [RS_7] + <-Map 1 [XPROD_EDGE] llap + XPROD_EDGE [RS_6] + Select Operator [SEL_2] (rows=166 width=178) + Output:["_col0","_col1"] + Filter Operator [FIL_16] (rows=166 width=178) + predicate:(key < 10) + TableScan [TS_0] (rows=500 width=178) + default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] + <-Map 4 [XPROD_EDGE] llap + XPROD_EDGE [RS_7] Select Operator [SEL_5] (rows=166 width=178) Output:["_col0","_col1"] Filter Operator [FIL_17] (rows=166 width=178) predicate:(key < 10) TableScan [TS_3] (rows=500 width=178) default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] - <-Select Operator [SEL_2] (rows=166 width=178) - Output:["_col0","_col1"] - Filter Operator [FIL_16] (rows=166 width=178) - predicate:(key < 10) - TableScan [TS_0] (rows=500 width=178) - default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] PREHOOK: query: explain select key, (c_int+1)+2 as x, sum(c_int) from cbo_t1 group by c_float, cbo_t1.c_int, key PREHOOK: type: QUERY http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out index 5194bc1..02dc49d 100644 --- a/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/hybridgrace_hashjoin_1.q.out @@ -1264,7 +1264,7 @@ POSTHOOK: Lineage: decimal_mapjoin.cdecimal1 EXPRESSION [(alltypesorc)alltypesor POSTHOOK: Lineage: decimal_mapjoin.cdecimal2 EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] POSTHOOK: Lineage: decimal_mapjoin.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1284,7 +1284,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1299,30 +1299,13 @@ STAGE PLANS: expressions: cdecimal1 (type: decimal(20,10)) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 551 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col2 - input vertices: - 1 Map 2 - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 551 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: r @@ -1340,6 +1323,28 @@ STAGE PLANS: value expressions: _col0 (type: decimal(23,14)) Execution mode: vectorized, llap LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col2 + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1347,7 +1352,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1363,8 +1368,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_mapjoin #### A masked pattern was here #### 6981 6981 NULL NULL -6981 6981 NULL -617.56077692307690 -6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1372,14 +1375,13 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL NULL 6981 6981 NULL -617.56077692307690 6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1388,8 +1390,8 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1398,13 +1400,14 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL NULL 6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL -617.56077692307690 +6981 6981 NULL -617.56077692307690 6981 6981 5831542.2692483780 NULL -6981 6981 5831542.2692483780 -617.56077692307690 -6981 6981 5831542.2692483780 -617.56077692307690 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL @@ -1412,9 +1415,9 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL +6981 6981 5831542.2692483780 -617.56077692307690 +6981 6981 5831542.2692483780 -617.56077692307690 6981 6981 NULL NULL -6981 6981 NULL -617.56077692307690 -6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1422,14 +1425,13 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL NULL 6981 6981 NULL -617.56077692307690 6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1438,13 +1440,14 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL NULL 6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL -617.56077692307690 +6981 6981 NULL -617.56077692307690 6981 6981 -515.6210729730 NULL -6981 6981 -515.6210729730 -617.56077692307690 -6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL @@ -1452,17 +1455,19 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL +6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 6984454.21109769200000 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +6981 6981 -515.6210729730 -617.56077692307690 +6981 6981 -515.6210729730 -617.56077692307690 +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: EXPLAIN SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1482,7 +1487,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -1497,30 +1502,13 @@ STAGE PLANS: expressions: cdecimal1 (type: decimal(20,10)) outputColumnNames: _col0 Statistics: Num rows: 5 Data size: 551 Basic stats: COMPLETE Column stats: NONE - Map Join Operator - condition map: - Inner Join 0 to 1 - keys: - 0 - 1 - outputColumnNames: _col0, _col2 - input vertices: - 1 Map 2 - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 551 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(20,10)) Execution mode: vectorized, llap LLAP IO: all inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: r @@ -1538,6 +1526,28 @@ STAGE PLANS: value expressions: _col0 (type: decimal(23,14)) Execution mode: vectorized, llap LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col2 + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 6981 (type: int), 6981 (type: int), _col0 (type: decimal(20,10)), _col2 (type: decimal(23,14)) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 25 Data size: 5535 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -1545,7 +1555,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[13][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[13][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: SELECT l.cint, r.cint, l.cdecimal1, r.cdecimal2 FROM decimal_mapjoin l JOIN decimal_mapjoin r ON l.cint = r.cint @@ -1561,8 +1571,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_mapjoin #### A masked pattern was here #### 6981 6981 NULL NULL -6981 6981 NULL -617.56077692307690 -6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1570,14 +1578,13 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL NULL 6981 6981 NULL -617.56077692307690 6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1586,8 +1593,8 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1596,13 +1603,14 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL NULL 6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL -617.56077692307690 +6981 6981 NULL -617.56077692307690 6981 6981 5831542.2692483780 NULL -6981 6981 5831542.2692483780 -617.56077692307690 -6981 6981 5831542.2692483780 -617.56077692307690 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL @@ -1610,9 +1618,9 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL 6981 6981 5831542.2692483780 NULL +6981 6981 5831542.2692483780 -617.56077692307690 +6981 6981 5831542.2692483780 -617.56077692307690 6981 6981 NULL NULL -6981 6981 NULL -617.56077692307690 -6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1620,14 +1628,13 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL NULL 6981 6981 NULL -617.56077692307690 6981 6981 NULL -617.56077692307690 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL -6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL +6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL @@ -1636,13 +1643,14 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL NULL 6981 6981 NULL 6984454.21109769200000 6981 6981 NULL NULL 6981 6981 NULL NULL 6981 6981 NULL NULL +6981 6981 NULL -617.56077692307690 +6981 6981 NULL -617.56077692307690 6981 6981 -515.6210729730 NULL -6981 6981 -515.6210729730 -617.56077692307690 -6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL @@ -1650,16 +1658,18 @@ POSTHOOK: Input: default@decimal_mapjoin 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL -6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 -617.56077692307690 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL +6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 6984454.21109769200000 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL 6981 6981 -515.6210729730 NULL +6981 6981 -515.6210729730 -617.56077692307690 +6981 6981 -515.6210729730 -617.56077692307690 PREHOOK: query: DROP TABLE decimal_mapjoin PREHOOK: type: DROPTABLE PREHOOK: Input: default@decimal_mapjoin http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out index bb507ce..6e0451f 100644 --- a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out +++ b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out @@ -117,7 +117,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Reducer 5 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Reducer 5 (XPROD_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) Reducer 5 <- Map 4 (SIMPLE_EDGE) #### A masked pattern was here #### http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/join0.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/join0.q.out b/ql/src/test/results/clientpositive/llap/join0.q.out index 1b23c3d..c2c6515 100644 --- a/ql/src/test/results/clientpositive/llap/join0.q.out +++ b/ql/src/test/results/clientpositive/llap/join0.q.out @@ -24,7 +24,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 4 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 4 (XPROD_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) #### A masked pattern was here #### Vertices: http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/leftsemijoin.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/leftsemijoin.q.out b/ql/src/test/results/clientpositive/llap/leftsemijoin.q.out index 611d929..9e6ad01 100644 --- a/ql/src/test/results/clientpositive/llap/leftsemijoin.q.out +++ b/ql/src/test/results/clientpositive/llap/leftsemijoin.q.out @@ -123,7 +123,7 @@ STAGE PLANS: #### A masked pattern was here #### Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) - Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE), Map 5 (CUSTOM_SIMPLE_EDGE) + Reducer 4 <- Map 3 (XPROD_EDGE), Map 5 (XPROD_EDGE) #### A masked pattern was here #### Vertices: Map 1 http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/mapjoin2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mapjoin2.q.out b/ql/src/test/results/clientpositive/llap/mapjoin2.q.out index ce65c6d..d43ec8e 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin2.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin2.q.out @@ -53,7 +53,7 @@ POSTHOOK: Input: default@tbl #### A masked pattern was here #### false false true true true true false false -Warning: Map Join MAPJOIN[9][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select a.key, a.a_one, b.b_one, a.a_zero, b.b_zero from ( SELECT 11 key, 0 confuse_you, 1 a_one, 0 a_zero ) a join ( SELECT 11 key, 0 confuse_you, 1 b_one, 0 b_zero ) b on a.key = b.key PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table http://git-wip-us.apache.org/repos/asf/hive/blob/cfbe6125/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out index ef26e92..505524e 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_hint.q.out @@ -428,7 +428,7 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (CUSTOM_SIMPLE_EDGE), Reducer 4 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: @@ -523,7 +523,7 @@ STAGE PLANS: Processor Tree: ListSink -Warning: Map Join MAPJOIN[24][bigTable=?] in task 'Map 1' is a cross product +Warning: Shuffle Join MERGEJOIN[24][tables = [$hdt$_0, $hdt$_1, $hdt$_2]] in Stage 'Reducer 2' is a cross product PREHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) PREHOOK: type: QUERY POSTHOOK: query: explain select * from part where p_name = (select p_name from part_null where p_name is null) @@ -537,8 +537,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Map 1 <- Map 2 (BROADCAST_EDGE), Reducer 3 (BROADCAST_EDGE) - Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE) + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE), Reducer 4 (XPROD_EDGE) + Reducer 4 <- Map 3 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -554,33 +554,13 @@ STAGE PLANS: expressions: p_partkey (type: int), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Inner Join 0 to 1 - Inner Join 0 to 2 - keys: - 0 - 1 - 2 - outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - input vertices: - 1 Reducer 3 - 2 Map 2 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 582 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: int), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) Execution mode: llap LLAP IO: no inputs - Map 2 + Map 3 Map Operator Tree: TableScan alias: part_null @@ -605,7 +585,31 @@ STAGE PLANS: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: no inputs - Reducer 3 + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + Inner Join 0 to 2 + keys: + 0 + 1 + 2 + outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), null (type: string), _col2 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: int), _col6 (type: string), _col7 (type: double), _col8 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 959 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 Execution mode: llap Reduce Operator Tree: Group By Operator
