Author: szehon
Date: Mon Dec 22 19:49:05 2014
New Revision: 1647389
URL: http://svn.apache.org/r1647389
Log:
HIVE-9173 : Fix auto_join29.q, auto_join_without_localtask.q determinism (Jimmy
Xiang via Szehon)
Modified:
hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q
hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
Modified: hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q Mon Dec 22
19:49:05 2014
@@ -1,4 +1,7 @@
set hive.auto.convert.join = true;
+
+-- SORT_QUERY_RESULTS
+
explain
SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value;
Modified:
hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
(original)
+++ hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
Mon Dec 22 19:49:05 2014
@@ -17,13 +17,13 @@ select a.* from src a join src b on a.ke
set hive.auto.convert.join.noconditionaltask.size=100;
explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40;
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40;
set hive.mapjoin.localtask.max.memory.usage = 0.0001;
set hive.mapjoin.check.memory.rows = 2;
-- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40;
Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out Mon Dec 22
19:49:05 2014
@@ -1,7 +1,11 @@
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND
src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key =
src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value,
src3.key, src3.value
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -1952,6 +1956,28 @@ POSTHOOK: query: SELECT * FROM src src1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 2 val_2 2 val_2
+NULL NULL 4 val_4 4 val_4
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 8 val_8 8 val_8
+NULL NULL 9 val_9 9 val_9
NULL NULL NULL NULL 10 val_10
NULL NULL NULL NULL 100 val_100
NULL NULL NULL NULL 100 val_100
@@ -2442,28 +2468,6 @@ NULL NULL NULL NULL 97 val_97
NULL NULL NULL NULL 97 val_97
NULL NULL NULL NULL 98 val_98
NULL NULL NULL NULL 98 val_98
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 2 val_2 2 val_2
-NULL NULL 4 val_4 4 val_4
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 8 val_8 8 val_8
-NULL NULL 9 val_9 9 val_9
PREHOOK: query: explain
SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10
AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND
src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key,
src3.value
PREHOOK: type: QUERY
Modified:
hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
(original)
+++
hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
Mon Dec 22 19:49:05 2014
@@ -612,30 +612,31 @@ RUN: Stage-3:MAPRED
103 val_103
103 val_103
PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-9 is a root stage , consists of Stage-12, Stage-13, Stage-2
- Stage-12 has a backup stage: Stage-2
- Stage-7 depends on stages: Stage-12
- Stage-6 depends on stages: Stage-2, Stage-7, Stage-8 , consists of Stage-4,
Stage-11, Stage-1
- Stage-4 has a backup stage: Stage-1
- Stage-11 has a backup stage: Stage-1
- Stage-5 depends on stages: Stage-11
- Stage-1
- Stage-13 has a backup stage: Stage-2
+ Stage-10 is a root stage , consists of Stage-13, Stage-14, Stage-3
+ Stage-13 has a backup stage: Stage-3
Stage-8 depends on stages: Stage-13
- Stage-2
- Stage-0 depends on stages: Stage-4, Stage-5, Stage-1
+ Stage-7 depends on stages: Stage-3, Stage-8, Stage-9 , consists of Stage-5,
Stage-12, Stage-1
+ Stage-5 has a backup stage: Stage-1
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
+ Stage-12 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-12
+ Stage-1
+ Stage-14 has a backup stage: Stage-3
+ Stage-9 depends on stages: Stage-14
+ Stage-3
+ Stage-0 depends on stages: Stage-2
STAGE PLANS:
- Stage: Stage-9
+ Stage: Stage-10
Conditional Operator
- Stage: Stage-12
+ Stage: Stage-13
Map Reduce Local Work
Alias -> Map Local Tables:
$hdt$_0:$hdt$_1:$hdt$_2:a
@@ -655,7 +656,7 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
- Stage: Stage-7
+ Stage: Stage-8
Map Reduce
Map Operator Tree:
TableScan
@@ -681,10 +682,10 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-6
+ Stage: Stage-7
Conditional Operator
- Stage: Stage-4
+ Stage: Stage-5
Map Reduce
Map Operator Tree:
TableScan
@@ -704,14 +705,12 @@ STAGE PLANS:
Select Operator
expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Limit
- Number of rows: 40
- File Output Operator
- compressed: false
- table:
- input format:
org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Local Work:
Map Reduce Local Work
Alias -> Map Local Tables:
@@ -722,7 +721,31 @@ STAGE PLANS:
$hdt$_0:$INTNAME
TableScan
- Stage: Stage-11
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1
(type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE
Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE
Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-12
Map Reduce Local Work
Alias -> Map Local Tables:
$hdt$_0:$hdt$_0:a
@@ -742,7 +765,7 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col1 (type: string)
- Stage: Stage-5
+ Stage: Stage-6
Map Reduce
Map Operator Tree:
TableScan
@@ -756,14 +779,12 @@ STAGE PLANS:
Select Operator
expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Limit
- Number of rows: 40
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Local Work:
Map Reduce Local Work
@@ -805,18 +826,14 @@ STAGE PLANS:
expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE
Column stats: NONE
- Limit
- Number of rows: 40
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE
Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE
Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
- Stage: Stage-13
+ Stage: Stage-14
Map Reduce Local Work
Alias -> Map Local Tables:
$hdt$_0:$hdt$_1:$hdt$_1:a
@@ -836,7 +853,7 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
- Stage: Stage-8
+ Stage: Stage-9
Map Reduce
Map Operator Tree:
TableScan
@@ -862,7 +879,7 @@ STAGE PLANS:
Local Work:
Map Reduce Local Work
- Stage: Stage-2
+ Stage: Stage-3
Map Reduce
Map Operator Tree:
TableScan
@@ -918,62 +935,63 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on
a.value=c.value where a.key>100 limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on
a.value=c.value where a.key>100 order by a.key, a.value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on
a.value=c.value where a.key>100 limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on
a.value=c.value where a.key>100 order by a.key, a.value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-RUN: Stage-9:CONDITIONAL
+RUN: Stage-10:CONDITIONAL
+RUN: Stage-14:MAPREDLOCAL
+RUN: Stage-9:MAPRED
+RUN: Stage-7:CONDITIONAL
RUN: Stage-12:MAPREDLOCAL
-RUN: Stage-7:MAPRED
-RUN: Stage-6:CONDITIONAL
-RUN: Stage-11:MAPREDLOCAL
-RUN: Stage-5:MAPRED
-238 val_238
-238 val_238
-238 val_238
-238 val_238
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-165 val_165
-165 val_165
-165 val_165
-165 val_165
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-255 val_255
-255 val_255
-255 val_255
-255 val_255
-278 val_278
-278 val_278
-278 val_278
-278 val_278
-484 val_484
-265 val_265
-265 val_265
-265 val_265
-265 val_265
-193 val_193
+RUN: Stage-6:MAPRED
+RUN: Stage-2:MAPRED
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+105 val_105
+111 val_111
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
PREHOOK: query: -- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
@@ -982,16 +1000,17 @@ ATTEMPT: Execute BackupTask: org.apache.
FAILED: Execution Error, return code 3 from
org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
POSTHOOK: query: -- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value
where a.key>100 order by a.key, a.value limit 40
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
#### A masked pattern was here ####
-RUN: Stage-9:CONDITIONAL
+RUN: Stage-10:CONDITIONAL
+RUN: Stage-14:MAPREDLOCAL
+RUN: Stage-3:MAPRED
+RUN: Stage-7:CONDITIONAL
RUN: Stage-12:MAPREDLOCAL
-RUN: Stage-2:MAPRED
-RUN: Stage-6:CONDITIONAL
-RUN: Stage-11:MAPREDLOCAL
RUN: Stage-1:MAPRED
+RUN: Stage-2:MAPRED
103 val_103
103 val_103
103 val_103
