[02/13] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/e8f283cf/ql/src/test/results/clientpositive/perf/tez/query8.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/tez/query8.q.out b/ql/src/test/results/clientpositive/perf/tez/query8.q.out index fef9ae9..e0742f2 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query8.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query8.q.out @@ -231,14 +231,14 @@ Stage-0 limit:100 Stage-1 Reducer 5 vectorized - File Output Operator [FS_107] -Limit [LIM_106] (rows=100 width=88) + File Output Operator [FS_115] +Limit [LIM_114] (rows=100 width=88) Number of rows:100 - Select Operator [SEL_105] (rows=348477374 width=88) + Select Operator [SEL_113] (rows=348477374 width=88) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] vectorized -SHUFFLE [RS_104] - Group By Operator [GBY_103] (rows=348477374 width=88) +SHUFFLE [RS_112] + Group By Operator [GBY_111] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_57] @@ -251,43 +251,43 @@ Stage-0 SHUFFLE [RS_53] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_88] (rows=1874 width=1911) - Conds:RS_99.substr(_col0, 1, 2)=RS_102.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] + Conds:RS_107.substr(_col0, 1, 2)=RS_110.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] <-Map 17 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_102] + SHUFFLE [RS_110] PartitionCols:substr(_col2, 1, 2) -Select Operator [SEL_101] (rows=1704 width=1910) +Select Operator [SEL_109] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_100] (rows=1704 width=1910) + Filter Operator [FIL_108] (rows=1704 width=1910) predicate:(s_store_sk is not null and substr(s_zip, 1, 2) is not null) TableScan [TS_42] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] <-Reducer 10 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_99] + SHUFFLE [RS_107] PartitionCols:substr(_col0, 1, 2) -Select Operator [SEL_98] (rows=1 width=1014) +Select Operator [SEL_106] (rows=1 width=1014) Output:["_col0"] - Filter Operator [FIL_97] (rows=1 width=1014) + Filter Operator [FIL_105] (rows=1 width=1014) predicate:(_col1 = 2L) -Group By Operator [GBY_96] (rows=683 width=1014) +Group By Operator [GBY_104] (rows=683 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Union 9 [SIMPLE_EDGE] <-Reducer 15 [CONTAINS] vectorized -Reduce Output Operator [RS_128] +Reduce Output Operator [RS_136] PartitionCols:_col0 - Group By Operator [GBY_127] (rows=1366 width=1014) + Group By Operator [GBY_135] (rows=1366 width=1014) Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 -Group By Operator [GBY_126] (rows=366 width=1014) +Group By Operator [GBY_134] (rows=366 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Reducer 14 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_125] + SHUFFLE [RS_133] PartitionCols:_col0 -Group By Operator [GBY_124] (rows=733 width=1014) +Group By Operator [GBY_132] (rows=733 width=1014)
[08/13] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/e8f283cf/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out index 66b6bd8..cdb9193 100644 --- a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out @@ -1,159 +1,159 @@ -PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: query: drop table small_alltypesorc1a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: query: drop table small_alltypesorc1a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: query: drop table small_alltypesorc2a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: query: drop table small_alltypesorc2a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: query: drop table small_alltypesorc3a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: query: drop table small_alltypesorc3a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: query: drop table small_alltypesorc4a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: query: drop table small_alltypesorc4a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: query: drop table small_alltypesorc_a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: query: drop table small_alltypesorc_a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: query: create table small_alltypesorc1a_n2 as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default -PREHOOK: Output: default@small_alltypesorc1a -POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: Output: default@small_alltypesorc1a_n2 +POSTHOOK: query: create table small_alltypesorc1a_n2 as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default -POSTHOOK: Output: default@small_alltypesorc1a -POSTHOOK: Lineage: small_alltypesorc1a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: create table small_alltypesorc2a as select * from allty
hive git commit: HIVE-19237: addendum to fix HIVE-19789
Repository: hive Updated Branches: refs/heads/master a4c5b412e -> 9f54302a8 HIVE-19237: addendum to fix HIVE-19789 Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f54302a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f54302a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f54302a Branch: refs/heads/master Commit: 9f54302a83460cbc3f04656cbb1f097c59dd3ec5 Parents: a4c5b41 Author: Zoltan Haindrich Authored: Mon Jun 11 18:41:23 2018 +0200 Committer: Zoltan Haindrich Committed: Mon Jun 11 18:41:42 2018 +0200 -- ql/src/test/results/clientpositive/llap/orc_llap.q.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/9f54302a/ql/src/test/results/clientpositive/llap/orc_llap.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/orc_llap.q.out b/ql/src/test/results/clientpositive/llap/orc_llap.q.out index 6bc6d4d..9235fbc 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap.q.out @@ -81,7 +81,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: default@cross_numbers POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers @@ -657,7 +657,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_llap A masked pattern was here -735462183586256 -Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +Warning: Shuffle Join MERGEJOIN[17][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product PREHOOK: query: insert into table orc_llap select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 from alltypesorc cross join cross_numbers
[12/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out index 6ec4d00..0c6bd6a 100644 --- a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out @@ -63,11 +63,11 @@ Stage-0 Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 4 [SIMPLE_EDGE] <-Reducer 3 [CONTAINS] llap -Reduce Output Operator [RS_53] +Reduce Output Operator [RS_84] PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=66/61 width=177) + Select Operator [SEL_82] (rows=66/61 width=177) Output:["_col0","_col1"] -Merge Join Operator [MERGEJOIN_79] (rows=66/61 width=177) +Merge Join Operator [MERGEJOIN_81] (rows=66/61 width=177) Conds:RS_21._col2=RS_22._col1(Inner),Output:["_col1","_col4"] <-Reducer 11 [SIMPLE_EDGE] llap SHUFFLE [RS_22] @@ -78,22 +78,22 @@ Stage-0 Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 10 [SIMPLE_EDGE] <-Map 12 [CONTAINS] llap - Reduce Output Operator [RS_15] + Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 -Select Operator [SEL_11] (rows=500/500 width=178) +Select Operator [SEL_96] (rows=500/500 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_72] (rows=500/500 width=178) + Filter Operator [FIL_95] (rows=500/500 width=178) predicate:value is not null -TableScan [TS_9] (rows=500/500 width=178) +TableScan [TS_94] (rows=500/500 width=178) Output:["key","value"] <-Map 9 [CONTAINS] llap - Reduce Output Operator [RS_15] + Reduce Output Operator [RS_93] PartitionCols:_col1, _col0 -Select Operator [SEL_8] (rows=25/25 width=175) +Select Operator [SEL_91] (rows=25/25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_71] (rows=25/25 width=175) + Filter Operator [FIL_90] (rows=25/25 width=175) predicate:value is not null -TableScan [TS_6] (rows=25/25 width=175) +TableScan [TS_89] (rows=25/25 width=175) Output:["key","value"] <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_21] @@ -119,11 +119,11 @@ Stage-0 TableScan [TS_3] (rows=25/25 width=175) default@src1,x,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"] <-Reducer 7 [CONTAINS] llap -Reduce Output Operator [RS_53] +Reduce Output Operator [RS_88] PartitionCols:_col0, _col1 - Select Operator [SEL_49] (rows=66/61 width=177) + Select Operator [SEL_86] (rows=66/61 width=177) Output:["_col0","_col1"] -Merge Join Operator [MERGEJOIN_80] (rows=66/61 width=177) +Merge Join Operator [MERGEJOIN_85] (rows=66/61 width=177) Conds:RS_46._col2=RS_47._col1(Inner),Output:["_col1","_col4"] <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_47] @@ -134,22 +134,22 @@ Stage-0 Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 14 [SIMPLE_EDGE] <-Map 13 [CONTAINS] llap - Reduce Output Operator [RS_40] + Reduce Output Operator [RS_103] PartitionCols:_col1, _col0 -Select Operator [SEL_33] (rows=25/25 width=175) +Select Operator [SEL_101] (rows=25/25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_75] (rows=25/25 width=175) + Filter Operator [FIL_100] (rows=25/25 width=175) predicate:value is not null -TableScan [TS_31] (rows=25/25 width=175) +Tab
[03/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query75.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/tez/query75.q.out b/ql/src/test/results/clientpositive/perf/tez/query75.q.out index 2611dce..ffb2a9d 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query75.q.out @@ -169,12 +169,12 @@ Stage-0 limit:-1 Stage-1 Reducer 10 vectorized - File Output Operator [FS_325] -Select Operator [SEL_324] (rows=100 width=111) + File Output Operator [FS_363] +Select Operator [SEL_362] (rows=100 width=111) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9"] - Limit [LIM_323] (rows=100 width=111) + Limit [LIM_361] (rows=100 width=111) Number of rows:100 -Select Operator [SEL_322] (rows=70276244 width=111) +Select Operator [SEL_360] (rows=70276244 width=111) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] <-Reducer 9 [SIMPLE_EDGE] SHUFFLE [RS_175] @@ -183,38 +183,38 @@ Stage-0 Filter Operator [FIL_173] (rows=70276244 width=111) predicate:((CAST( _col10 AS decimal(17,2)) / CAST( _col4 AS decimal(17,2))) < 0.9) Merge Join Operator [MERGEJOIN_282] (rows=210828734 width=111) - Conds:RS_318._col0, _col1, _col2, _col3=RS_321._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] + Conds:RS_356._col0, _col1, _col2, _col3=RS_359._col0, _col1, _col2, _col3(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col10","_col11"] <-Reducer 17 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_321] + SHUFFLE [RS_359] PartitionCols:_col0, _col1, _col2, _col3 -Group By Operator [GBY_320] (rows=191662482 width=111) +Group By Operator [GBY_358] (rows=191662482 width=111) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3 - Group By Operator [GBY_319] (rows=383324964 width=111) + Group By Operator [GBY_357] (rows=383324964 width=111) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 <-Union 16 [SIMPLE_EDGE] <-Reducer 15 [CONTAINS] vectorized - Reduce Output Operator [RS_328] + Reduce Output Operator [RS_366] PartitionCols:_col0, _col1, _col2, _col3 -Group By Operator [GBY_327] (rows=766649929 width=111) +Group By Operator [GBY_365] (rows=766649929 width=111) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Group By Operator [GBY_326] (rows=574982367 width=103) + Group By Operator [GBY_364] (rows=574982367 width=103) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4, KEY._col5 <-Union 14 [SIMPLE_EDGE] <-Reducer 13 [CONTAINS] - Reduce Output Operator [RS_133] + Reduce Output Operator [RS_296] PartitionCols:_col0, _col1, _col2, _col3, _col4, _col5 -Group By Operator [GBY_132] (rows=1149964734 width=103) +Group By Operator [GBY_295] (rows=1149964734 width=103) Output:["_col0","_col1","_col2","_col3","_col4","_col5"],keys:_col0, _col1, _col2, _col3, _col4, _col5 - Select Operator [SEL_106] (rows=383314495 width=135) + Select Operator [SEL_293] (rows=383314495 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] -Merge Join Operator [MERGEJOIN_275] (rows=383314495 width=135) - Conds:RS_103._col1, _col2=RS_312._col0, _col1(Left Outer),Output:["_col3","_col4","_col8"
[14/14] hive git commit: HIVE-16505: Support "unknown" boolean truth value (Laszlo Bodor via Zoltan Haindrich)
HIVE-16505: Support "unknown" boolean truth value (Laszlo Bodor via Zoltan
Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f37fef06
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f37fef06
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f37fef06
Branch: refs/heads/master
Commit: f37fef06f37a4bc5931eb6c9e63b48d2ae1182a6
Parents: 2842f88
Author: Laszlo Bodor
Authored: Mon Jun 11 08:31:37 2018 +0200
Committer: Zoltan Haindrich
Committed: Mon Jun 11 08:31:37 2018 +0200
--
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 1 +
.../apache/hadoop/hive/ql/parse/HiveParser.g| 2 +
.../hadoop/hive/ql/parse/IdentifiersParser.g| 5 +-
.../test/queries/clientpositive/bool_unknown.q | 10 +++
.../results/clientpositive/bool_unknown.q.out | 65
5 files changed, 82 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f37fef06/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index d4b5782..43ad7dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
KW_TRUE : 'TRUE';
KW_FALSE : 'FALSE';
+KW_UNKNOWN : 'UNKNOWN';
KW_ALL : 'ALL';
KW_NONE: 'NONE';
KW_AND : 'AND';
http://git-wip-us.apache.org/repos/asf/hive/blob/f37fef06/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index d4a0ed3..324c804 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -69,6 +69,7 @@ TOK_OP_NOT;
TOK_OP_LIKE;
TOK_TRUE;
TOK_FALSE;
+TOK_UNKNOWN;
TOK_TRANSFORM;
TOK_SERDE;
TOK_SERDENAME;
@@ -451,6 +452,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
// Keywords
xlateMap.put("KW_TRUE", "TRUE");
xlateMap.put("KW_FALSE", "FALSE");
+xlateMap.put("KW_UNKNOWN", "UNKNOWN");
xlateMap.put("KW_ALL", "ALL");
xlateMap.put("KW_NONE", "NONE");
xlateMap.put("KW_AND", "AND");
http://git-wip-us.apache.org/repos/asf/hive/blob/f37fef06/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index 55bd92d..7dc6146 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -450,9 +450,11 @@ isCondition
: KW_NULL -> Identifier["isnull"]
| KW_TRUE -> Identifier["istrue"]
| KW_FALSE -> Identifier["isfalse"]
+| KW_UNKNOWN -> Identifier["isnull"]
| KW_NOT KW_NULL -> Identifier["isnotnull"]
| KW_NOT KW_TRUE -> Identifier["isnottrue"]
| KW_NOT KW_FALSE -> Identifier["isnotfalse"]
+| KW_NOT KW_UNKNOWN -> Identifier["isnotnull"]
;
precedenceUnaryPrefixExpression
@@ -663,7 +665,7 @@ precedenceOrExpression
booleanValue
:
-KW_TRUE^ | KW_FALSE^
+KW_TRUE^ | KW_FALSE^ | KW_UNKNOWN^
;
booleanValueTok
@@ -835,6 +837,7 @@ nonReserved
| KW_REOPTIMIZATION
| KW_RESOURCE | KW_PLAN | KW_PLANS | KW_QUERY_PARALLELISM | KW_ACTIVATE |
KW_MOVE | KW_DO
| KW_POOL | KW_ALLOC_FRACTION | KW_SCHEDULING_POLICY | KW_PATH |
KW_MAPPING | KW_WORKLOAD | KW_MANAGEMENT | KW_ACTIVE | KW_UNMANAGED
+| KW_UNKNOWN
;
http://git-wip-us.apache.org/repos/asf/hive/blob/f37fef06/ql/src/test/queries/clientpositive/bool_unknown.q
--
diff --git a/ql/src/test/queries/clientpositive/bool_unknown.q
b/ql/src/test/queries/clientpositive/bool_unknown.q
new file mode 100644
index 000..159bab3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/bool_unknown.q
@@ -0,0 +1,10 @@
+select assert_true(null is unknown);
+
+select assert_true(true is not unknown);
+select assert_true(false is not unknown);
+
+select assert_true((null = null) is unknown);
+select assert_true((null = false) is unknown);
+select assert_true((null = true) is unknown);
+
+select assert_true((select cast(null as boolean)) is unknown);
http://git-wip-us.apache.org/repos/asf/hive/blob/f37fef06/ql/src/test/results/clientpositive/bool_unknown.q.out
--
[09/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/llap/lineage3.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/lineage3.q.out
b/ql/src/test/results/clientpositive/llap/lineage3.q.out
index 16c4efa..27dd874 100644
--- a/ql/src/test/results/clientpositive/llap/lineage3.q.out
+++ b/ql/src/test/results/clientpositive/llap/lineage3.q.out
@@ -25,7 +25,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc
PREHOOK: Output: default@d1
PREHOOK: Output: default@d2
-{"version":"1.0","engine":"tez","database":"default","hash":"84e3cdc38011da5842162df175b2a494","queryText":"from
(select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on
a.cint = b.cbigint) t_n20\ninsert into table d1 select x where y is
null\ninsert into table d2 select y where x >
0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1],"expression":"a.cint
is not
null","edgeType":"PREDICATE"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint)
=
b.cbigint)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.cbigint
is not
null","edgeType":"PREDICATE"},{"sources":[5],"targets":[0],"expression":"t_n20.y
is
null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST(
y AS
varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[1],"expression":"(t_n20.x
>
0Y)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0],"expression":"compute
_stats(UDFToInteger(x),
'hll')","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
+{"version":"1.0","engine":"tez","database":"default","hash":"84e3cdc38011da5842162df175b2a494","queryText":"from
(select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on
a.cint = b.cbigint) t_n20\ninsert into table d1 select x where y is
null\ninsert into table d2 select y where x >
0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1],"expression":"a.cint
is not
null","edgeType":"PREDICATE"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint)
=
b.cbigint)","edgeType":"PREDICATE"},{"sources":[4],"targets":[0,1],"expression":"b.cbigint
is not
null","edgeType":"PREDICATE"},{"sources":[5],"targets":[0],"expression":"t_n20.y
is
null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST(
y AS
varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[1,0],"expression":"(t_n20.x
> 0Y)","edgeType":"PREDICATE"},{"sources":[2],"targets":[0],"expression":"compu
te_stats(UDFToInteger(x),
'hll')","edgeType":"PROJECTION"},{"sources":[5],"targets":[0],"expression":"compute_stats(CAST(
y AS varchar(128)),
'hll')","edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
PREHOOK: query: drop table if exists t_n20
PREHOOK: type: DROPTABLE
PREHOOK: query: create table t_n20 as
@@ -372,7 +372,7 @@ PREHOOK: query: create table src_dp1 (f string, w string, m
int)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@src_dp1
-Warning: Shuffle Join MERGEJOIN[19][tables = [$hdt$_0, $hdt$_1]] in Stage
'Reducer 2' is a cross product
+Warning: Shuffle Join MERGEJOIN[54][tables = [$hdt$_0, $hdt$_1]] in Stage
'Reducer 2' is a cross product
PREHOOK: query: from src_dp, src_dp1
insert into dest_dp1 partition (year) select first, word, year
insert into dest_dp2 partition (y, m) select first, word, year, month
@@ -387,4 +387,7 @@ PREHOOK: Output: default@dest_dp1@year=0
PREHOOK: Output: default@dest_dp2
PREHOOK: Output: default@dest_dp2@y=1
PREHOOK: Output: default@dest_dp3@y=2
-{"version":"1.0","engine":"tez","database":"default","hash":"e540a88155ffa4bf6842a4fdf3bfe639","queryText":"from
src_dp, src_dp1\ninsert into dest_dp1 partition (year) select first, word,
year\ninsert into dest_dp2 partition (y, m) select first, word, year,
month\ninsert into dest_dp3 partition (y=2, m, d) select first, word, month m,
day d where year=2\ninsert into dest_dp2 partition (y=1, m) select f, w,
m\ninsert into dest_dp1 partition (
[01/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 24a98ea24 -> f37fef06f
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
--
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
index c253fd2..f32397a 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_1.q.out
@@ -22,20 +22,20 @@ Stage-0
Stage-1
Union 2
<-Map 1 [CONTAINS]
-File Output Operator [FS_7]
- Limit [LIM_6] (rows=10/20 width=178)
+File Output Operator [FS_12]
+ Limit [LIM_11] (rows=10/10 width=178)
Number of rows:10
-Select Operator [SEL_1] (rows=500/12 width=178)
+Select Operator [SEL_9] (rows=500/12 width=178)
Output:["_col0","_col1"]
- TableScan [TS_0] (rows=500/13 width=178)
+ TableScan [TS_8] (rows=500/13 width=178)
Output:["key","value"]
<-Map 3 [CONTAINS]
-File Output Operator [FS_7]
- Limit [LIM_6] (rows=10/20 width=178)
+File Output Operator [FS_17]
+ Limit [LIM_16] (rows=10/10 width=178)
Number of rows:10
-Select Operator [SEL_3] (rows=500/12 width=178)
+Select Operator [SEL_14] (rows=500/12 width=178)
Output:["_col0","_col1"]
- TableScan [TS_2] (rows=500/13 width=178)
+ TableScan [TS_13] (rows=500/13 width=178)
Output:["key","value"]
PREHOOK: query: select key from src
@@ -132,8 +132,8 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 2
- File Output Operator [FS_5]
-Group By Operator [GBY_3] (rows=1/1 width=440)
+ File Output Operator [FS_8]
+Group By Operator [GBY_6] (rows=1/1 width=440)
Output:["_col0"],aggregations:["compute_stats(VALUE._col0,
'hll')"]
<-Map 1 [CUSTOM_SIMPLE_EDGE]
File Output Operator [FS_2]
@@ -142,8 +142,8 @@ Stage-3
Output:["_col0"]
TableScan [TS_0] (rows=500/500 width=87)
default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
- PARTITION_ONLY_SHUFFLE [RS_2]
-Select Operator [SEL_1] (rows=500/500 width=87)
+ PARTITION_ONLY_SHUFFLE [RS_5]
+Select Operator [SEL_4] (rows=500/500 width=87)
Output:["key"]
Please refer to the previous Select Operator [SEL_1]
@@ -401,16 +401,16 @@ Stage-0
Stage-1
Union 2
<-Map 1 [CONTAINS]
-File Output Operator [FS_6]
- Select Operator [SEL_1] (rows=500/500 width=178)
+File Output Operator [FS_10]
+ Select Operator [SEL_8] (rows=500/500 width=178)
Output:["_col0","_col1"]
-TableScan [TS_0] (rows=500/500 width=178)
+TableScan [TS_7] (rows=500/500 width=178)
Output:["key","value"]
<-Map 3 [CONTAINS]
-File Output Operator [FS_6]
- Select Operator [SEL_3] (rows=500/500 width=178)
+File Output Operator [FS_14]
+ Select Operator [SEL_12] (rows=500/500 width=178)
Output:["_col0","_col1"]
-TableScan [TS_2] (rows=500/500 width=178)
+TableScan [TS_11] (rows=500/500 width=178)
Output:["key","value"]
PREHOOK: query: select count(*) from (select * from src a union all select *
from src b)subq
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
--
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index 42bad01..a6eb4e5 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -669,22 +669,22 @@ Stage-3
Conditional Operator
Stage-1
Reducer 2
- File Output Operator [FS_5]
-Group By Operator [GBY_3] (rows=1/1 width=2760)
+ File Output Operator [FS_9]
+Group By Operator [GBY_7] (rows=1/1 width=2760)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["compute_stats(VALUE._col0,
'hll')","compute_stats(VALUE._col2, 'hll')","compute_stats(VALUE._col3,
'hll')","compute_stats(VALUE._col4, 'hll')","compute_stats(VALUE._col5, 'hll')"]
<-Map 1 [CUSTOM_SIM
[06/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query2.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/tez/query2.q.out b/ql/src/test/results/clientpositive/perf/tez/query2.q.out index 61776db..f964747 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query2.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query2.q.out @@ -133,8 +133,8 @@ Stage-0 limit:-1 Stage-1 Reducer 7 vectorized - File Output Operator [FS_112] -Select Operator [SEL_111] (rows=287491028 width=135) + File Output Operator [FS_122] +Select Operator [SEL_121] (rows=287491028 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] <-Reducer 6 [SIMPLE_EDGE] SHUFFLE [RS_58] @@ -146,20 +146,20 @@ Stage-0 FORWARD [RS_54] PartitionCols:_col0 Merge Join Operator [MERGEJOIN_92] (rows=261355475 width=135) - Conds:RS_103._col0=RS_109._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] + Conds:RS_113._col0=RS_119._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] <-Map 11 [SIMPLE_EDGE] vectorized -SHUFFLE [RS_109] +SHUFFLE [RS_119] PartitionCols:_col0 - Select Operator [SEL_107] (rows=36524 width=1119) + Select Operator [SEL_117] (rows=36524 width=1119) Output:["_col0"] -Filter Operator [FIL_105] (rows=36524 width=1119) +Filter Operator [FIL_115] (rows=36524 width=1119) predicate:((d_year = 2001) and d_week_seq is not null) TableScan [TS_20] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_week_seq","d_year"] <-Reducer 4 [ONE_TO_ONE_EDGE] vectorized -FORWARD [RS_103] +FORWARD [RS_113] PartitionCols:_col0 - Group By Operator [GBY_102] (rows=237595882 width=135) + Group By Operator [GBY_112] (rows=237595882 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)","sum(VALUE._col4)","sum(VALUE._col5)","sum(VALUE._col6)"],keys:KEY._col0 <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_17] @@ -169,50 +169,50 @@ Stage-0 Select Operator [SEL_14] (rows=475191764 width=135) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7"] Merge Join Operator [MERGEJOIN_91] (rows=475191764 width=135) -Conds:Union 2._col0=RS_101._col0(Inner),Output:["_col1","_col3","_col4"] +Conds:Union 2._col0=RS_111._col0(Inner),Output:["_col1","_col3","_col4"] <-Map 10 [SIMPLE_EDGE] vectorized -SHUFFLE [RS_101] +SHUFFLE [RS_111] PartitionCols:_col0 - Select Operator [SEL_100] (rows=73049 width=1119) + Select Operator [SEL_110] (rows=73049 width=1119) Output:["_col0","_col1","_col2"] -Filter Operator [FIL_99] (rows=73049 width=1119) +Filter Operator [FIL_109] (rows=73049 width=1119) predicate:(d_date_sk is not null and d_week_seq is not null) TableScan [TS_8] (rows=73049 width=1119) default@date_dim,date_dim,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_week_seq","d_day_name"] <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] vectorized - Reduce Output Operator [RS_98] + Reduce Output Operator [RS_108] PartitionCols:_col0 -Select Operator [SEL_97] (rows=144002668 width=135) +Select Operator [SEL_107] (rows=144002668 width=135) Output:["_col0","_col1"] - Filter Operator [FIL_96] (rows=144002668 width=135) + Filter Operator [FIL_106]
[08/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out index 66b6bd8..cdb9193 100644 --- a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out @@ -1,159 +1,159 @@ -PREHOOK: query: drop table small_alltypesorc1a +PREHOOK: query: drop table small_alltypesorc1a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc1a +POSTHOOK: query: drop table small_alltypesorc1a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc2a +PREHOOK: query: drop table small_alltypesorc2a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc2a +POSTHOOK: query: drop table small_alltypesorc2a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc3a +PREHOOK: query: drop table small_alltypesorc3a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc3a +POSTHOOK: query: drop table small_alltypesorc3a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc4a +PREHOOK: query: drop table small_alltypesorc4a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc4a +POSTHOOK: query: drop table small_alltypesorc4a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: drop table small_alltypesorc_a +PREHOOK: query: drop table small_alltypesorc_a_n2 PREHOOK: type: DROPTABLE -POSTHOOK: query: drop table small_alltypesorc_a +POSTHOOK: query: drop table small_alltypesorc_a_n2 POSTHOOK: type: DROPTABLE -PREHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: query: create table small_alltypesorc1a_n2 as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@alltypesorc PREHOOK: Output: database:default -PREHOOK: Output: default@small_alltypesorc1a -POSTHOOK: query: create table small_alltypesorc1a as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 +PREHOOK: Output: default@small_alltypesorc1a_n2 +POSTHOOK: query: create table small_alltypesorc1a_n2 as select * from alltypesorc where cint is not null and ctinyint is not null order by ctinyint, csmallint, cint, cbigint, cfloat, cdouble, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2 limit 5 POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default -POSTHOOK: Output: default@small_alltypesorc1a -POSTHOOK: Lineage: small_alltypesorc1a.cbigint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cbigint, type:bigint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cboolean1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean1, type:boolean, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cboolean2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cboolean2, type:boolean, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.csmallint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.cstring2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring2, type:string, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctimestamp2 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp2, type:timestamp, comment:null), ] -POSTHOOK: Lineage: small_alltypesorc1a.ctinyint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctinyint, type:tinyint, comment:null), ] -PREHOOK: query: create table small_alltypesorc2a as select * from allty
[10/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/llap/explainuser_2.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out index 735915e..ea32d34 100644 --- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out @@ -460,124 +460,124 @@ Stage-0 limit:-1 Stage-1 Reducer 5 vectorized, llap - File Output Operator [FS_96] -Group By Operator [GBY_95] (rows=33 width=177) + File Output Operator [FS_130] +Group By Operator [GBY_129] (rows=33 width=177) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 4 [SIMPLE_EDGE] <-Reducer 3 [CONTAINS] llap -Reduce Output Operator [RS_56] +Reduce Output Operator [RS_88] PartitionCols:_col0, _col1 - Group By Operator [GBY_55] (rows=33 width=177) + Group By Operator [GBY_87] (rows=33 width=177) Output:["_col0","_col1"],keys:_col0, _col1 -Select Operator [SEL_25] (rows=33 width=177) +Select Operator [SEL_85] (rows=33 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_82] (rows=33 width=177) - Conds:RS_22._col2=RS_94._col1(Inner),Output:["_col1","_col4"] + Merge Join Operator [MERGEJOIN_84] (rows=33 width=177) + Conds:RS_22._col2=RS_128._col1(Inner),Output:["_col1","_col4"] <-Reducer 11 [SIMPLE_EDGE] vectorized, llap -SHUFFLE [RS_94] +SHUFFLE [RS_128] PartitionCols:_col1 - Select Operator [SEL_93] (rows=262 width=178) + Select Operator [SEL_127] (rows=262 width=178) Output:["_col1"] -Group By Operator [GBY_92] (rows=262 width=178) +Group By Operator [GBY_126] (rows=262 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 10 [SIMPLE_EDGE] <-Map 12 [CONTAINS] vectorized, llap -Reduce Output Operator [RS_107] +Reduce Output Operator [RS_141] PartitionCols:_col0, _col1 - Group By Operator [GBY_106] (rows=262 width=178) + Group By Operator [GBY_140] (rows=262 width=178) Output:["_col0","_col1"],keys:_col1, _col0 -Select Operator [SEL_105] (rows=500 width=178) +Select Operator [SEL_139] (rows=500 width=178) Output:["_col0","_col1"] - Filter Operator [FIL_104] (rows=500 width=178) + Filter Operator [FIL_138] (rows=500 width=178) predicate:value is not null -TableScan [TS_9] (rows=500 width=178) +TableScan [TS_100] (rows=500 width=178) Output:["key","value"] <-Map 9 [CONTAINS] vectorized, llap -Reduce Output Operator [RS_103] +Reduce Output Operator [RS_137] PartitionCols:_col0, _col1 - Group By Operator [GBY_102] (rows=262 width=178) + Group By Operator [GBY_136] (rows=262 width=178) Output:["_col0","_col1"],keys:_col1, _col0 -Select Operator [SEL_101] (rows=25 width=175) +Select Operator [SEL_135] (rows=25 width=175) Output:["_col0","_col1"] - Filter Operator [FIL_100] (rows=25 width=175) + Filter Operator [FIL_134] (rows=25 width=175) predicate:value is not null -TableScan [TS_6] (rows=25 width=175) +TableScan [TS_94] (rows=25 width=175) Output:["key","value"] <-Reducer 2 [SIMPLE_EDGE] llap SHUFFLE [RS_22] PartitionCols:_col2 Merge Join Operator [MERGEJOIN_80] (rows=39 width=175) - Conds:RS_86._col0=RS_90._col0(Inner),Output:["_col1","_col2"] + Conds:RS_120._col0=RS_124._col0(Inner),Output:["_col1","_col2
[05/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query49.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/tez/query49.q.out
b/ql/src/test/results/clientpositive/perf/tez/query49.q.out
index 3a9c640..47f37a2 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query49.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query49.q.out
@@ -277,44 +277,44 @@ Stage-0
limit:100
Stage-1
Reducer 11 vectorized
- File Output Operator [FS_179]
-Limit [LIM_178] (rows=100 width=101)
+ File Output Operator [FS_205]
+Limit [LIM_204] (rows=100 width=101)
Number of rows:100
- Select Operator [SEL_177] (rows=5915494 width=101)
+ Select Operator [SEL_203] (rows=5915494 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 10 [SIMPLE_EDGE] vectorized
-SHUFFLE [RS_176]
- Select Operator [SEL_175] (rows=5915494 width=101)
+SHUFFLE [RS_202]
+ Select Operator [SEL_201] (rows=5915494 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
-Group By Operator [GBY_174] (rows=5915494 width=101)
+Group By Operator [GBY_200] (rows=5915494 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"],keys:KEY._col0, KEY._col1,
KEY._col2, KEY._col3, KEY._col4
<-Union 9 [SIMPLE_EDGE]
<-Reducer 22 [CONTAINS] vectorized
-Reduce Output Operator [RS_215]
+Reduce Output Operator [RS_241]
PartitionCols:_col0, _col1, _col2, _col3, _col4
- Group By Operator [GBY_214] (rows=11830988 width=101)
+ Group By Operator [GBY_240] (rows=11830988 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"],keys:_col0, _col3, _col4,
_col1, _col2
-Select Operator [SEL_213] (rows=8604378 width=88)
+Select Operator [SEL_239] (rows=8604378 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_212] (rows=8604378 width=88)
+ Filter Operator [FIL_238] (rows=8604378 width=88)
predicate:((_col0 <= 10) or (rank_window_1 <= 10))
-PTF Operator [PTF_211] (rows=12906568 width=88)
+PTF Operator [PTF_237] (rows=12906568 width=88)
Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col4 AS
decimal(15,4)) / CAST( _col5 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
- Select Operator [SEL_210] (rows=12906568
width=88)
+ Select Operator [SEL_236] (rows=12906568
width=88)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Reducer 21 [SIMPLE_EDGE] vectorized
-SHUFFLE [RS_209]
+SHUFFLE [RS_235]
PartitionCols:0
- Select Operator [SEL_208] (rows=12906568
width=88)
+ Select Operator [SEL_234] (rows=12906568
width=88)
Output:["rank_window_0","_col0","_col1","_col2","_col3","_col4"]
-PTF Operator [PTF_207] (rows=12906568
width=88)
+PTF Operator [PTF_233] (rows=12906568
width=88)
Function
definitions:[{},{"name:":"windowingtablefunction","order by:":"(CAST( _col1 AS
decimal(15,4)) / CAST( _col2 AS decimal(15,4))) ASC NULLS FIRST","partition
by:":"0"}]
- Select Operator [SEL_206] (rows=12906568
width=88)
+ Select Operator [SEL_232] (rows=12906568
width=88)
Output:["_col0","_col1","_col2","_col3","_col4"]
<-Reducer 20 [SIMPLE_EDGE] vectorized
-SHUFFLE [RS_205]
+SHUFFLE [RS_231]
PartitionCols:0
- Group By Operator [GBY_204]
(rows=12906568 width=88)
+ Group By Operator [GBY_230]
(rows=12906568 width=88)
Output:["_col0","_col1","_col2","_col3","_col4"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)","sum(VALUE._col3)"],keys:KEY._col0
<-Reduc
[02/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query8.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/tez/query8.q.out b/ql/src/test/results/clientpositive/perf/tez/query8.q.out index fef9ae9..e0742f2 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query8.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query8.q.out @@ -231,14 +231,14 @@ Stage-0 limit:100 Stage-1 Reducer 5 vectorized - File Output Operator [FS_107] -Limit [LIM_106] (rows=100 width=88) + File Output Operator [FS_115] +Limit [LIM_114] (rows=100 width=88) Number of rows:100 - Select Operator [SEL_105] (rows=348477374 width=88) + Select Operator [SEL_113] (rows=348477374 width=88) Output:["_col0","_col1"] <-Reducer 4 [SIMPLE_EDGE] vectorized -SHUFFLE [RS_104] - Group By Operator [GBY_103] (rows=348477374 width=88) +SHUFFLE [RS_112] + Group By Operator [GBY_111] (rows=348477374 width=88) Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0 <-Reducer 3 [SIMPLE_EDGE] SHUFFLE [RS_57] @@ -251,43 +251,43 @@ Stage-0 SHUFFLE [RS_53] PartitionCols:_col1 Merge Join Operator [MERGEJOIN_88] (rows=1874 width=1911) - Conds:RS_99.substr(_col0, 1, 2)=RS_102.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] + Conds:RS_107.substr(_col0, 1, 2)=RS_110.substr(_col2, 1, 2)(Inner),Output:["_col1","_col2"] <-Map 17 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_102] + SHUFFLE [RS_110] PartitionCols:substr(_col2, 1, 2) -Select Operator [SEL_101] (rows=1704 width=1910) +Select Operator [SEL_109] (rows=1704 width=1910) Output:["_col0","_col1","_col2"] - Filter Operator [FIL_100] (rows=1704 width=1910) + Filter Operator [FIL_108] (rows=1704 width=1910) predicate:(s_store_sk is not null and substr(s_zip, 1, 2) is not null) TableScan [TS_42] (rows=1704 width=1910) default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_name","s_zip"] <-Reducer 10 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_99] + SHUFFLE [RS_107] PartitionCols:substr(_col0, 1, 2) -Select Operator [SEL_98] (rows=1 width=1014) +Select Operator [SEL_106] (rows=1 width=1014) Output:["_col0"] - Filter Operator [FIL_97] (rows=1 width=1014) + Filter Operator [FIL_105] (rows=1 width=1014) predicate:(_col1 = 2L) -Group By Operator [GBY_96] (rows=683 width=1014) +Group By Operator [GBY_104] (rows=683 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Union 9 [SIMPLE_EDGE] <-Reducer 15 [CONTAINS] vectorized -Reduce Output Operator [RS_128] +Reduce Output Operator [RS_136] PartitionCols:_col0 - Group By Operator [GBY_127] (rows=1366 width=1014) + Group By Operator [GBY_135] (rows=1366 width=1014) Output:["_col0","_col1"],aggregations:["count(_col1)"],keys:_col0 -Group By Operator [GBY_126] (rows=366 width=1014) +Group By Operator [GBY_134] (rows=366 width=1014) Output:["_col0","_col1"],aggregations:["count(VALUE._col0)"],keys:KEY._col0 <-Reducer 14 [SIMPLE_EDGE] vectorized - SHUFFLE [RS_125] + SHUFFLE [RS_133] PartitionCols:_col0 -Group By Operator [GBY_124] (rows=733 width=1014) +Group By Operator [GBY_132] (rows=733 width=1014)
[07/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query14.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/tez/query14.q.out b/ql/src/test/results/clientpositive/perf/tez/query14.q.out index 6366d14..4d27064 100644 --- a/ql/src/test/results/clientpositive/perf/tez/query14.q.out +++ b/ql/src/test/results/clientpositive/perf/tez/query14.q.out @@ -1,6 +1,6 @@ -Warning: Shuffle Join MERGEJOIN[893][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product -Warning: Shuffle Join MERGEJOIN[894][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product -Warning: Shuffle Join MERGEJOIN[895][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product +Warning: Shuffle Join MERGEJOIN[902][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 5' is a cross product +Warning: Shuffle Join MERGEJOIN[914][tables = [$hdt$_1, $hdt$_2, $hdt$_0]] in Stage 'Reducer 12' is a cross product +Warning: Shuffle Join MERGEJOIN[926][tables = [$hdt$_2, $hdt$_3, $hdt$_1]] in Stage 'Reducer 16' is a cross product PREHOOK: query: explain with cross_items as (select i_item_sk ss_item_sk @@ -281,203 +281,203 @@ Stage-0 limit:100 Stage-1 Reducer 8 vectorized - File Output Operator [FS_972] -Limit [LIM_971] (rows=100 width=237) + File Output Operator [FS_1134] +Limit [LIM_1133] (rows=100 width=237) Number of rows:100 - Select Operator [SEL_970] (rows=1016388080 width=237) + Select Operator [SEL_1132] (rows=1016388080 width=237) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] <-Reducer 7 [SIMPLE_EDGE] vectorized -SHUFFLE [RS_969] - Select Operator [SEL_968] (rows=1016388080 width=237) +SHUFFLE [RS_1131] + Select Operator [SEL_1130] (rows=1016388080 width=237) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] -Group By Operator [GBY_967] (rows=1016388080 width=237) +Group By Operator [GBY_1129] (rows=1016388080 width=237) Output:["_col0","_col1","_col2","_col3","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3, KEY._col4 <-Union 6 [SIMPLE_EDGE] <-Reducer 12 [CONTAINS] -Reduce Output Operator [RS_571] +Reduce Output Operator [RS_919] PartitionCols:_col0, _col1, _col2, _col3, _col4 - Group By Operator [GBY_570] (rows=2032776160 width=237) + Group By Operator [GBY_918] (rows=2032776160 width=237) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col4)","sum(_col5)"],keys:_col0, _col1, _col2, _col3, 0L -Select Operator [SEL_377] (rows=116155905 width=264) +Select Operator [SEL_916] (rows=116155905 width=264) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - Filter Operator [FIL_376] (rows=116155905 width=264) + Filter Operator [FIL_915] (rows=116155905 width=264) predicate:(_col5 > _col1) -Merge Join Operator [MERGEJOIN_894] (rows=348467716 width=264) +Merge Join Operator [MERGEJOIN_914] (rows=348467716 width=264) Conds:(Inner),(Inner),Output:["_col1","_col2","_col3","_col4","_col5","_col6"] <-Reducer 11 [CUSTOM_SIMPLE_EDGE] vectorized - PARTITION_ONLY_SHUFFLE [RS_978] -Select Operator [SEL_977] (rows=1 width=8) - Filter Operator [FIL_976] (rows=1 width=8) + PARTITION_ONLY_SHUFFLE [RS_1140] +Select Operator [SEL_1139] (rows=1 width=8) + Filter Operator [FIL_1138] (rows=1 width=8) predicate:(sq_count_check(_col0) <= 1) -Group By Operator [GBY_975] (rows=1 width=8) +Group By Operator [GBY_1137] (rows=1 width=8) Output:["_col0"],aggregations:["count()"] - Select Operator [SEL_974] (rows=1 width=8) -Group By Operator [GBY_973] (rows=1 width=8) + Select Operator [SEL_1136] (rows=1 width=8) +Group By Operator [GBY_1135] (rows=1 width=8) Output:["_col0
[04/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/perf/tez/query56.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/tez/query56.q.out
b/ql/src/test/results/clientpositive/perf/tez/query56.q.out
index 80514cf..13657a8 100644
--- a/ql/src/test/results/clientpositive/perf/tez/query56.q.out
+++ b/ql/src/test/results/clientpositive/perf/tez/query56.q.out
@@ -157,22 +157,22 @@ Stage-0
limit:100
Stage-1
Reducer 7 vectorized
- File Output Operator [FS_214]
-Limit [LIM_213] (rows=100 width=108)
+ File Output Operator [FS_226]
+Limit [LIM_225] (rows=100 width=108)
Number of rows:100
- Select Operator [SEL_212] (rows=335408073 width=108)
+ Select Operator [SEL_224] (rows=335408073 width=108)
Output:["_col0","_col1"]
<-Reducer 6 [SIMPLE_EDGE] vectorized
-SHUFFLE [RS_211]
- Group By Operator [GBY_210] (rows=335408073 width=108)
+SHUFFLE [RS_223]
+ Group By Operator [GBY_222] (rows=335408073 width=108)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
<-Union 5 [SIMPLE_EDGE]
<-Reducer 11 [CONTAINS] vectorized
- Reduce Output Operator [RS_226]
+ Reduce Output Operator [RS_238]
PartitionCols:_col0
-Group By Operator [GBY_225] (rows=670816147 width=108)
+Group By Operator [GBY_237] (rows=670816147 width=108)
Output:["_col0","_col1"],aggregations:["sum(_col1)"],keys:_col0
- Group By Operator [GBY_224] (rows=95833781 width=135)
+ Group By Operator [GBY_236] (rows=95833781 width=135)
Output:["_col0","_col1"],aggregations:["sum(VALUE._col0)"],keys:KEY._col0
<-Reducer 10 [SIMPLE_EDGE]
SHUFFLE [RS_109]
@@ -185,29 +185,29 @@ Stage-0
SHUFFLE [RS_104]
PartitionCols:_col0
Merge Join Operator [MERGEJOIN_173]
(rows=508200 width=1436)
-
Conds:RS_187._col1=RS_193._col0(Inner),Output:["_col0","_col1"]
+
Conds:RS_199._col1=RS_205._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_187]
+ SHUFFLE [RS_199]
PartitionCols:_col1
-Select Operator [SEL_186] (rows=462000
width=1436)
+Select Operator [SEL_198] (rows=462000
width=1436)
Output:["_col0","_col1"]
- Filter Operator [FIL_185] (rows=462000
width=1436)
+ Filter Operator [FIL_197] (rows=462000
width=1436)
predicate:(i_item_id is not null and
i_item_sk is not null)
TableScan [TS_0] (rows=462000
width=1436)
default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id"]
<-Reducer 13 [ONE_TO_ONE_EDGE] vectorized
- FORWARD [RS_193]
+ FORWARD [RS_205]
PartitionCols:_col0
-Group By Operator [GBY_192] (rows=115500
width=1436)
+Group By Operator [GBY_204] (rows=115500
width=1436)
Output:["_col0"],keys:KEY._col0
<-Map 12 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_191]
+ SHUFFLE [RS_203]
PartitionCols:_col0
-Group By Operator [GBY_190]
(rows=231000 width=1436)
+Group By Operator [GBY_202]
(rows=231000 width=1436)
Output:["_col0"],keys:i_item_id
- Select Operator [SEL_189]
(rows=231000 width=1436)
+ Select Operator [SEL_201]
(rows=231000 width=1436)
Output:["i_item_id"]
-Filter Operator [FIL_188]
(rows=231000 width=1436)
+Filter Operator [FIL_200]
(rows=231000 width=1436)
predicate:((i_color) IN
('orchid', 'ch
[13/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2842f88e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2842f88e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2842f88e Branch: refs/heads/master Commit: 2842f88e228bb4f11e587fb41338c9a770c96262 Parents: 24a98ea Author: Zoltan Haindrich Authored: Mon Jun 11 08:01:49 2018 +0200 Committer: Zoltan Haindrich Committed: Mon Jun 11 08:01:49 2018 +0200 -- data/conf/llap/hive-site.xml|2 +- .../java/org/apache/hadoop/hive/ql/Context.java |5 + .../apache/hadoop/hive/ql/exec/Operator.java| 23 +- .../hadoop/hive/ql/exec/OperatorUtils.java |4 +- .../hive/ql/exec/SerializationUtilities.java| 14 +- .../ql/hooks/NoOperatorReuseCheckerHook.java| 109 ++ .../ql/optimizer/physical/MemoryDecider.java| 12 +- .../ql/optimizer/physical/SerializeFilter.java |6 +- .../ql/parse/ColumnStatsAutoGatherContext.java |1 + .../hive/ql/parse/spark/SplitOpTreeForDPP.java |7 +- .../hadoop/hive/ql/plan/MergeJoinWork.java |7 +- .../ql/plan/mapping/TestCounterMapping.java |8 +- .../hive/ql/plan/mapping/TestOperatorCmp.java | 16 +- .../results/clientpositive/auto_join0.q.out |8 +- .../cbo_rp_cross_product_check_2.q.out |4 +- .../clientpositive/cross_product_check_2.q.out |4 +- .../llap/bucketizedhiveinputformat.q.out|2 +- .../clientpositive/llap/constprog_dpp.q.out | 28 +- .../clientpositive/llap/dp_counter_mm.q.out | 172 +- .../clientpositive/llap/dp_counter_non_mm.q.out | 172 +- .../llap/dynamic_semijoin_user_level.q.out | 16 +- .../clientpositive/llap/explainanalyze_2.q.out | 672 +++ .../clientpositive/llap/explainuser_1.q.out | 128 +- .../clientpositive/llap/explainuser_2.q.out | 1840 +- .../results/clientpositive/llap/lineage3.q.out |9 +- .../llap/table_access_keys_stats.q.out |2 +- .../llap/tez_input_counters.q.out | 14 +- .../clientpositive/llap/union_fast_stats.q.out | 624 +++--- .../test/results/clientpositive/mapjoin47.q.out |4 +- .../clientpositive/perf/spark/query77.q.out |2 +- .../clientpositive/perf/tez/query14.q.out | 746 +++ .../clientpositive/perf/tez/query2.q.out| 52 +- .../clientpositive/perf/tez/query23.q.out | 196 +- .../clientpositive/perf/tez/query33.q.out | 106 +- .../clientpositive/perf/tez/query38.q.out | 106 +- .../clientpositive/perf/tez/query49.q.out | 154 +- .../clientpositive/perf/tez/query5.q.out| 140 +- .../clientpositive/perf/tez/query54.q.out | 160 +- .../clientpositive/perf/tez/query56.q.out | 106 +- .../clientpositive/perf/tez/query60.q.out | 106 +- .../clientpositive/perf/tez/query66.q.out | 96 +- .../clientpositive/perf/tez/query71.q.out | 86 +- .../clientpositive/perf/tez/query75.q.out | 236 +-- .../clientpositive/perf/tez/query76.q.out | 96 +- .../clientpositive/perf/tez/query77.q.out | 168 +- .../clientpositive/perf/tez/query8.q.out| 84 +- .../clientpositive/perf/tez/query80.q.out | 162 +- .../clientpositive/perf/tez/query87.q.out | 132 +- .../results/clientpositive/smb_mapjoin_25.q.out | 16 +- .../results/clientpositive/smb_mapjoin_47.q.out |4 +- .../spark/spark_explainuser_1.q.out | 64 +- .../clientpositive/subquery_multiinsert.q.out | 12 +- .../clientpositive/tez/explainanalyze_1.q.out | 36 +- .../clientpositive/tez/explainanalyze_3.q.out | 16 +- .../clientpositive/tez/explainanalyze_5.q.out | 28 +- .../clientpositive/tez/explainuser_3.q.out | 12 +- .../clientpositive/vector_groupby_mapjoin.q.out |8 +- 57 files changed, 3621 insertions(+), 3422 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/data/conf/llap/hive-site.xml -- diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 1507a56..44ca6c9 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -163,7 +163,7 @@ hive.exec.post.hooks - org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook + org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook, org.apache.hadoop.hive.ql.hooks.NoOperatorReuseCheckerHook Post Execute Hook for Tests http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/sr
[11/14] hive git commit: HIVE-19237: Only use an operatorId once in a plan (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/2842f88e/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index e022634..7b99494 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -56,10 +56,10 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 2 llap
- File Output Operator [FS_6]
-Select Operator [SEL_5] (rows=1 width=1077)
+ File Output Operator [FS_10]
+Select Operator [SEL_9] (rows=1 width=1077)
Output:["_col0","_col1","_col2","_col3"]
- Group By Operator [GBY_4] (rows=1 width=1077)
+ Group By Operator [GBY_8] (rows=1 width=1077)
Output:["_col0","_col1","_col2","_col3"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"],keys:KEY._col0,
KEY._col1
<-Map 1 [SIMPLE_EDGE] llap
File Output Operator [FS_3]
@@ -68,11 +68,11 @@ Stage-3
Output:["_col0","_col1"]
TableScan [TS_0] (rows=500 width=178)
default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
-SHUFFLE [RS_3]
+SHUFFLE [RS_7]
PartitionCols:_col0, _col1
- Group By Operator [GBY_2] (rows=1 width=1061)
+ Group By Operator [GBY_6] (rows=1 width=1061)
Output:["_col0","_col1","_col2","_col3"],aggregations:["compute_stats(key,
'hll')","compute_stats(value, 'hll')"],keys:ds, ts
-Select Operator [SEL_1] (rows=500 width=292)
+Select Operator [SEL_5] (rows=500 width=292)
Output:["key","value","ds","ts"]
Please refer to the previous Select Operator [SEL_1]
@@ -105,10 +105,10 @@ Stage-3
Dependency Collection{}
Stage-1
Reducer 3 llap
- File Output Operator [FS_6]
-Select Operator [SEL_5] (rows=1 width=1077)
+ File Output Operator [FS_14]
+Select Operator [SEL_13] (rows=1 width=1077)
Output:["_col0","_col1","_col2","_col3"]
- Group By Operator [GBY_4] (rows=1 width=1077)
+ Group By Operator [GBY_12] (rows=1 width=1077)
Output:["_col0","_col1","_col2","_col3"],aggregations:["compute_stats(VALUE._col0)","compute_stats(VALUE._col1)"],keys:KEY._col0,
KEY._col1
<-Reducer 2 [SIMPLE_EDGE] llap
File Output Operator [FS_7]
@@ -127,11 +127,11 @@ Stage-3
Output:["_col0","_col1"]
TableScan [TS_0] (rows=500 width=178)
default@src,src,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
-SHUFFLE [RS_3]
+SHUFFLE [RS_11]
PartitionCols:_col0, _col1
- Group By Operator [GBY_2] (rows=1 width=1061)
+ Group By Operator [GBY_10] (rows=1 width=1061)
Output:["_col0","_col1","_col2","_col3"],aggregations:["compute_stats(key,
'hll')","compute_stats(value, 'hll')"],keys:ds, ts
-Select Operator [SEL_1] (rows=100 width=292)
+Select Operator [SEL_9] (rows=100 width=292)
Output:["key","value","ds","ts"]
Please refer to the previous Select Operator [SEL_6]
@@ -778,10 +778,10 @@ Stage-0
Output:["_col0"]
<-Union 3 [SIMPLE_EDGE]
<-Reducer 2 [CONTAINS] llap
-Reduce Output Operator [RS_23]
- Select Operator [SEL_5] (rows=1 width=87)
+Reduce Output Operator [RS_29]
+ Select Operator [SEL_27] (rows=1 width=87)
Output:["_col0"]
-Group By Operator [GBY_4] (rows=1 width=8)
+Group By Operator [GBY_26] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [CUSTOM_SIMPLE_EDGE] llap
PARTITION_ONLY_SHUFFLE [RS_3]
@@ -792,10 +792,10 @@ Stage-0
TableScan [TS_0] (rows=20 width=80)
default@cbo_t3,s1,Tbl:COMPLETE,Col:COMPLETE,Output:["key"]
<-Reducer 6 [CONTAINS] llap
-Reduce Output Operator [RS_23]
- Select Operator [SEL_12] (rows=1 width=87)
+Reduce Output Operator [RS_33]
+ Select Operator [SEL_31] (rows
[2/2] hive git commit: HIVE-18823: Vectorization: introduce qtest for SUM (IF/WHEN) with vectorization for ORC (Laszlo Bodor via Matt McCline)
HIVE-18823: Vectorization: introduce qtest for SUM (IF/WHEN) with vectorization for ORC (Laszlo Bodor via Matt McCline) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec0636c0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec0636c0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec0636c0 Branch: refs/heads/master Commit: ec0636c066d0348077fa872ef98e542e5e5b1b2c Parents: 4750e41 Author: Laszlo Bodor Authored: Thu Jun 7 10:17:10 2018 +0200 Committer: Zoltan Haindrich Committed: Thu Jun 7 10:17:10 2018 +0200 -- .../clientpositive/vectorization_sum_if_when.q | 19 +++ .../vectorization_sum_if_when.q.out | 126 +++ 2 files changed, 145 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/ec0636c0/ql/src/test/queries/clientpositive/vectorization_sum_if_when.q -- diff --git a/ql/src/test/queries/clientpositive/vectorization_sum_if_when.q b/ql/src/test/queries/clientpositive/vectorization_sum_if_when.q new file mode 100644 index 000..b70df7d --- /dev/null +++ b/ql/src/test/queries/clientpositive/vectorization_sum_if_when.q @@ -0,0 +1,19 @@ +set hive.vectorized.execution.enabled=false; +drop table if exists vectorization_sum_if_when_a; +drop table if exists vectorization_sum_if_when_b; +create table vectorization_sum_if_when_a (x int) stored as orc; +insert into table vectorization_sum_if_when_a values (0), (1), (0), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL); +create table vectorization_sum_if_when_b (x int) stored as orc; +insert into table vectorization_sum_if_when_b select least(t1.x + t2.x + t3.x + t4.x, 1) from vectorization_sum_if_when_a t1, vectorization_sum_if_when_a t2, vectorization_sum_if_when_a t3, vectorization_sum_if_when_a t4; +select count(*), x from vectorization_sum_if_when_b group by x; + +select sum(IF(x is null, 1, 0)), count(1) from vectorization_sum_if_when_b; +select sum(IF(x=1, 1, 0)), count(1) from vectorization_sum_if_when_b; +select sum((case WHEN x = 1 THEN 1 else 0 end)) from vectorization_sum_if_when_b; +select sum((case WHEN x = 1 THEN 1 else 0 end)), sum((case WHEN x = 1 THEN 1 when x is null then 0 else 0 end)) from vectorization_sum_if_when_b; + +set hive.vectorized.execution.enabled=true; +select sum(IF(x is null, 1, 0)), count(1) from vectorization_sum_if_when_b; +select sum(IF(x=1, 1, 0)), count(1) from vectorization_sum_if_when_b; +select sum((case WHEN x = 1 THEN 1 else 0 end)) from vectorization_sum_if_when_b; +select sum((case WHEN x = 1 THEN 1 else 0 end)), sum((case WHEN x = 1 THEN 1 when x is null then 0 else 0 end)) from vectorization_sum_if_when_b; http://git-wip-us.apache.org/repos/asf/hive/blob/ec0636c0/ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out -- diff --git a/ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out b/ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out new file mode 100644 index 000..61090dc --- /dev/null +++ b/ql/src/test/results/clientpositive/vectorization_sum_if_when.q.out @@ -0,0 +1,126 @@ +PREHOOK: query: drop table if exists vectorization_sum_if_when_a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists vectorization_sum_if_when_a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists vectorization_sum_if_when_b +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists vectorization_sum_if_when_b +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table vectorization_sum_if_when_a (x int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vectorization_sum_if_when_a +POSTHOOK: query: create table vectorization_sum_if_when_a (x int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vectorization_sum_if_when_a +PREHOOK: query: insert into table vectorization_sum_if_when_a values (0), (1), (0), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@vectorization_sum_if_when_a +POSTHOOK: query: insert into table vectorization_sum_if_when_a values (0), (1), (0), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL), (NULL) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@vectorization_sum_if_when_a +POSTHOOK: Lineage: vectorization_sum_if_when_a.x SCRIPT [] +PREHOOK: query: create table vectorization_sum_if_when_b (x int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@vectorization_sum_if_when_b +POSTHOOK: que
[1/2] hive git commit: HIVE-19727: Fix Signature matching of table aliases (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master fdd103dde -> ec0636c06
HIVE-19727: Fix Signature matching of table aliases (Zoltan Haindrich reviewed
by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4750e411
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4750e411
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4750e411
Branch: refs/heads/master
Commit: 4750e411b23ba297fb8fe0f752c5438c079b393e
Parents: fdd103d
Author: Zoltan Haindrich
Authored: Thu Jun 7 09:59:59 2018 +0200
Committer: Zoltan Haindrich
Committed: Thu Jun 7 09:59:59 2018 +0200
--
.../hadoop/hive/ql/plan/TableScanDesc.java | 24 ++--
.../signature/TestOperatorSignature.java| 8 +-
.../spark/dynamic_rdd_cache.q.out | 57 +---
.../results/clientpositive/spark/join22.q.out | 20 +--
.../spark/spark_explainuser_1.q.out | 44 ++-
.../clientpositive/spark/subquery_select.q.out | 21 +--
.../results/clientpositive/spark/union10.q.out | 38 +-
.../results/clientpositive/spark/union11.q.out | 44 +--
.../results/clientpositive/spark/union15.q.out | 48 ++-
.../results/clientpositive/spark/union2.q.out | 21 +--
.../results/clientpositive/spark/union20.q.out | 19 +--
.../results/clientpositive/spark/union25.q.out | 22 +---
.../results/clientpositive/spark/union4.q.out | 19 +--
.../results/clientpositive/spark/union5.q.out | 21 +--
.../results/clientpositive/spark/union9.q.out | 40 +-
.../clientpositive/spark/union_ppr.q.out| 132 +--
16 files changed, 52 insertions(+), 526 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/4750e411/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 57df7e2..192c04c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -52,10 +52,10 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
private List virtualCols;
private String statsAggKeyPrefix; // stats publishing/aggregating key
prefix
- /**
- * A list of the partition columns of the table.
- * Set by the semantic analyzer only in case of the analyze command.
- */
+ /**
+ * A list of the partition columns of the table.
+ * Set by the semantic analyzer only in case of the analyze command.
+ */
private List partColumns;
/**
@@ -92,13 +92,13 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
private transient List referencedColumns;
public static final String FILTER_EXPR_CONF_STR =
-"hive.io.filter.expr.serialized";
+ "hive.io.filter.expr.serialized";
public static final String FILTER_TEXT_CONF_STR =
-"hive.io.filter.text";
+ "hive.io.filter.text";
public static final String FILTER_OBJECT_CONF_STR =
-"hive.io.filter.object";
+ "hive.io.filter.object";
// input file name (big) to bucket number
private Map bucketFileNameMapping;
@@ -157,8 +157,6 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
}
@Explain(displayName = "alias")
- // FIXME: this might not needed to be in the signature; but in that case the
compare shouldn't consider it either!
- @Signature
public String getAlias() {
return alias;
}
@@ -386,9 +384,9 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
return isMetadataOnly;
}
- // @Signature
+ @Signature
public String getQualifiedTable() {
-return tableMetadata.getFullyQualifiedName();
+return dbName + "." + tableName;
}
public Table getTableMetadata() {
@@ -540,7 +538,7 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
public boolean isSame(OperatorDesc other) {
if (getClass().getName().equals(other.getClass().getName())) {
TableScanDesc otherDesc = (TableScanDesc) other;
- return Objects.equals(getAlias(), otherDesc.getAlias()) &&
+ return Objects.equals(getQualifiedTable(),
otherDesc.getQualifiedTable()) &&
ExprNodeDescUtils.isSame(getFilterExpr(), otherDesc.getFilterExpr())
&&
getRowLimit() == otherDesc.getRowLimit() &&
isGatherStats() == otherDesc.isGatherStats();
@@ -549,6 +547,6 @@ public class TableScanDesc extends AbstractOperatorDesc
implements IStatsGatherD
}
public boolean isFullAcidTable() {
- return isTranscationalTable() &&
!getAcidOperati
hive git commit: HIVE-19460: Improve stats estimations for NOT IN operator (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 32888e82c -> 603b0f64c
HIVE-19460: Improve stats estimations for NOT IN operator (Zoltan Haindrich
reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/603b0f64
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/603b0f64
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/603b0f64
Branch: refs/heads/branch-3
Commit: 603b0f64cb95dc381a9fa050c9d25ba4d709166d
Parents: 32888e8
Author: Zoltan Haindrich
Authored: Tue May 29 12:48:53 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue May 29 12:48:53 2018 +0200
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../stats/annotation/StatsRulesProcFactory.java | 158 ++-
.../hadoop/hive/ql/plan/ColStatistics.java | 1 -
.../ql/plan/mapping/TestStatEstimations.java| 113 +
.../clientpositive/llap/acid_no_buckets.q.out | 20 +--
.../clientpositive/llap/explainuser_2.q.out | 26 +--
.../clientpositive/llap/vector_between_in.q.out | 14 +-
.../clientpositive/llap/vector_struct_in.q.out | 6 +-
.../clientpositive/llap/vectorization_0.q.out | 16 +-
9 files changed, 312 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/603b0f64/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 72336ab..66c2831 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2373,6 +2373,8 @@ public class HiveConf extends Configuration {
"in the number of rows filtered by a certain operator, which in turn
might lead to overprovision or\n" +
"underprovision of resources. This factor is applied to the
cardinality estimation of IN clauses in\n" +
"filter operators."),
+HIVE_STATS_IN_MIN_RATIO("hive.stats.filter.in.min.ratio", (float) 0.05,
+"Output estimation of an IN filter can't be lower than this ratio"),
// Concurrency
HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false,
"Whether Hive supports concurrency control or not. \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/603b0f64/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 91cccfb..d0be33b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.optimizer.stats.annotation;
import java.lang.reflect.Field;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,7 +30,6 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.Stack;
-
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ColStatistics.Range;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -494,6 +494,19 @@ public class StatsRulesProcFactory {
}
}
+ boolean allColsFilteredByStats = true;
+ for (int i = 0; i < columnStats.size(); i++) {
+ValuePruner vp = new ValuePruner(columnStats.get(i));
+allColsFilteredByStats &= vp.isValid();
+Set newValues = Sets.newHashSet();
+for (ExprNodeDescEqualityWrapper v : values.get(i)) {
+ if (vp.accept(v)) {
+newValues.add(v);
+ }
+}
+values.set(i, newValues);
+ }
+
// 3. Calculate IN selectivity
double factor = 1d;
for (int i = 0; i < columnStats.size(); i++) {
@@ -503,10 +516,151 @@ public class StatsRulesProcFactory {
// max can be 1,
hive git commit: HIVE-19460: Improve stats estimations for NOT IN operator (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 9d23f7185 -> 99ed2bcbc
HIVE-19460: Improve stats estimations for NOT IN operator (Zoltan Haindrich
reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/99ed2bcb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/99ed2bcb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/99ed2bcb
Branch: refs/heads/master
Commit: 99ed2bcbcb408cbcd81e77a1ca76c50a3bd43260
Parents: 9d23f71
Author: Zoltan Haindrich
Authored: Tue May 29 12:47:25 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue May 29 12:47:25 2018 +0200
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../stats/annotation/StatsRulesProcFactory.java | 158 ++-
.../hadoop/hive/ql/plan/ColStatistics.java | 1 -
.../ql/plan/mapping/TestStatEstimations.java| 113 +
.../clientpositive/llap/acid_no_buckets.q.out | 20 +--
.../clientpositive/llap/explainuser_2.q.out | 26 +--
.../clientpositive/llap/vector_between_in.q.out | 14 +-
.../clientpositive/llap/vector_struct_in.q.out | 6 +-
.../clientpositive/llap/vectorization_0.q.out | 16 +-
9 files changed, 312 insertions(+), 44 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/99ed2bcb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index f48d004..7942608 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2374,6 +2374,8 @@ public class HiveConf extends Configuration {
"in the number of rows filtered by a certain operator, which in turn
might lead to overprovision or\n" +
"underprovision of resources. This factor is applied to the
cardinality estimation of IN clauses in\n" +
"filter operators."),
+HIVE_STATS_IN_MIN_RATIO("hive.stats.filter.in.min.ratio", (float) 0.05,
+"Output estimation of an IN filter can't be lower than this ratio"),
// Concurrency
HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false,
"Whether Hive supports concurrency control or not. \n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/99ed2bcb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 91cccfb..d0be33b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.optimizer.stats.annotation;
import java.lang.reflect.Field;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,7 +30,6 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.Stack;
-
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -60,6 +59,7 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
+import org.apache.hadoop.hive.ql.plan.ColStatistics.Range;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -494,6 +494,19 @@ public class StatsRulesProcFactory {
}
}
+ boolean allColsFilteredByStats = true;
+ for (int i = 0; i < columnStats.size(); i++) {
+ValuePruner vp = new ValuePruner(columnStats.get(i));
+allColsFilteredByStats &= vp.isValid();
+Set newValues = Sets.newHashSet();
+for (ExprNodeDescEqualityWrapper v : values.get(i)) {
+ if (vp.accept(v)) {
+newValues.add(v);
+ }
+}
+values.set(i, newValues);
+ }
+
// 3. Calculate IN selectivity
double factor = 1d;
for (int i = 0; i < columnStats.size(); i++) {
@@ -503,10 +516,151 @@ public class StatsRulesProcFactory {
// max can be 1, eve
hive git commit: HIVE-19713: itests/hive-jmh should not reference a concreate storage-api version (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 66e5453df -> 32888e82c
HIVE-19713: itests/hive-jmh should not reference a concreate storage-api
version (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/32888e82
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/32888e82
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/32888e82
Branch: refs/heads/branch-3
Commit: 32888e82c272dbc9f3f214807a4a05fad71d93b7
Parents: 66e5453
Author: Zoltan Haindrich
Authored: Tue May 29 08:26:39 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue May 29 08:26:39 2018 +0200
--
itests/hive-jmh/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/32888e82/itests/hive-jmh/pom.xml
--
diff --git a/itests/hive-jmh/pom.xml b/itests/hive-jmh/pom.xml
index 5eb3026..e045ace 100644
--- a/itests/hive-jmh/pom.xml
+++ b/itests/hive-jmh/pom.xml
@@ -66,7 +66,7 @@
org.apache.hive
hive-storage-api
- 2.7.0-SNAPSHOT
+ ${storage-api.version}
org.apache.hive
hive git commit: HIVE-19713: itests/hive-jmh should not reference a concreate storage-api version (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 7562cdb71 -> 83e15d824
HIVE-19713: itests/hive-jmh should not reference a concreate storage-api
version (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83e15d82
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83e15d82
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83e15d82
Branch: refs/heads/master
Commit: 83e15d8242c3bf265f823b82f81b31987ece6189
Parents: 7562cdb
Author: Zoltan Haindrich
Authored: Tue May 29 08:19:26 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue May 29 08:19:26 2018 +0200
--
itests/hive-jmh/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/83e15d82/itests/hive-jmh/pom.xml
--
diff --git a/itests/hive-jmh/pom.xml b/itests/hive-jmh/pom.xml
index de5e180..0abefdf 100644
--- a/itests/hive-jmh/pom.xml
+++ b/itests/hive-jmh/pom.xml
@@ -66,7 +66,7 @@
org.apache.hive
hive-storage-api
- 2.7.0-SNAPSHOT
+ ${storage-api.version}
org.apache.hive
[2/3] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/196c671a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out -- diff --git a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out new file mode 100644 index 000..8cc0fbc --- /dev/null +++ b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out @@ -0,0 +1,784 @@ +PREHOOK: query: drop table if exists t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists t8 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t8 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 (a integer,b integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 (a integer,b integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t8 like t1 +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8 like t1 +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1 +POSTHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.a SCRIPT [] +POSTHOOK: Lineage: t1.b SCRIPT [] +PREHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t8 +POSTHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t8.a EXPRESSION [(t1)t1.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: t8.b EXPRESSION [(t1)t1.FieldSchema(name:b, type:int, comment:null), ] +PREHOOK: query: analyze table t1 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 + A masked pattern was here +POSTHOOK: query: analyze table t1 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 + A masked pattern was here +PREHOOK: query: analyze table t8 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t8 +PREHOOK: Output: default@t8 + A masked pattern was here +POSTHOOK: query: analyze table t8 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t8 +POSTHOOK: Output: default@t8 + A masked pattern was here +PREHOOK: query: select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +PREHOOK: Input: default@t8 + A masked pattern was here +POSTHOOK: query: select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8 + A masked pattern was here +PREHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: t8 +Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE +Filter Operator + predicate: (b) IN (2, 3) (type: boolean) + Statistics: Num rows: 16/16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator +aggregations: sum(a) +keys: b (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: bigi
[3/3] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19500: Prevent multiple selectivity estimations for the same variable in
conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/196c671a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/196c671a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/196c671a
Branch: refs/heads/branch-3
Commit: 196c671a18693318fd37933a54f2f8b880c2301b
Parents: b7ec52e
Author: Zoltan Haindrich
Authored: Fri May 25 14:07:04 2018 +0200
Committer: Zoltan Haindrich
Committed: Fri May 25 14:08:20 2018 +0200
--
.../stats/annotation/StatsRulesProcFactory.java | 12 +-
.../clientpositive/stat_estimate_related_col.q | 43 +
.../annotate_stats_deep_filters.q.out | 4 +-
.../annotate_stats_join_pkfk.q.out | 48 +-
.../llap/bucketsortoptimize_insert_7.q.out | 20 +-
.../clientpositive/llap/explainuser_1.q.out | 64 +-
.../llap/orc_predicate_pushdown.q.out | 20 +-
.../llap/parquet_predicate_pushdown.q.out | 20 +-
.../clientpositive/llap/subquery_multi.q.out| 26 +-
.../llap/vector_decimal_cast.q.out | 4 +-
.../clientpositive/llap/vector_if_expr.q.out| 10 +-
.../llap/vector_mapjoin_reduce.q.out| 14 +-
.../clientpositive/llap/vectorization_0.q.out | 12 +-
.../llap/vectorization_short_regress.q.out | 8 +-
.../llap/vectorized_nested_mapjoin.q.out| 12 +-
.../spark/spark_explainuser_1.q.out | 68 +-
.../stat_estimate_related_col.q.out | 784 +++
17 files changed, 998 insertions(+), 171 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/196c671a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index c59128d..91cccfb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -349,12 +349,12 @@ public class StatsRulesProcFactory {
for (ExprNodeDesc child : genFunc.getChildren()) {
evaluatedRowCount = evaluateChildExpr(aspCtx.getAndExprStats(),
child,
aspCtx, neededCols, op, evaluatedRowCount);
- }
- newNumRows = evaluatedRowCount;
- if (satisfyPrecondition(aspCtx.getAndExprStats())) {
-updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
- } else {
-updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+newNumRows = evaluatedRowCount;
+if (satisfyPrecondition(aspCtx.getAndExprStats())) {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
+} else {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+}
}
} else if (udf instanceof GenericUDFOPOr) {
// for OR condition independently compute and update stats.
http://git-wip-us.apache.org/repos/asf/hive/blob/196c671a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
--
diff --git a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
new file mode 100644
index 000..52da2f7
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
@@ -0,0 +1,43 @@
+set hive.explain.user=true;
+set hive.strict.checks.cartesian.product=false;
+set hive.stats.fetch.column.stats=true;
+
+drop table if exists t1;
+drop table if exists t8;
+
+create table t1 (a integer,b integer);
+create table t8 like t1;
+
+insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+insert into t8
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1 union all
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1
+;
+
+analyze table t1 compute statistics for columns;
+analyze table t8 compute statistics for columns;
+
+explain analyze select sum(a) from t8 where b in (2,3) group by b;
+explain analyze select sum(a) from t8 where b=2 group by b;
+
+explain analyze select sum(a) from t1 where 2=b and b=2 group by b;
+
+explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b;
+explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b;
+
+
+explain analyze sele
[1/3] hive git commit: HIVE-19557: stats: filters for dates are not taking advantage of min/max values (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 a93d1bd66 -> 196c671a1
HIVE-19557: stats: filters for dates are not taking advantage of min/max values
(Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7ec52e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7ec52e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7ec52e3
Branch: refs/heads/branch-3
Commit: b7ec52e35045e4364952361d2865d28c4955d5e5
Parents: a93d1bd
Author: Zoltan Haindrich
Authored: Fri May 25 07:00:19 2018 +0200
Committer: Zoltan Haindrich
Committed: Fri May 25 14:07:50 2018 +0200
--
.../test/resources/testconfiguration.properties | 1 +
.../stats/annotation/StatsRulesProcFactory.java | 10 +-
.../clientpositive/colstats_date_min_max.q | 30 +++
.../llap/colstats_date_min_max.q.out| 193 +++
.../clientpositive/llap/vector_between_in.q.out | 20 +-
5 files changed, 243 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b7ec52e3/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 67b2441..39b83a2 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -489,6 +489,7 @@ minillaplocal.query.files=\
cbo_subq_not_in.q,\
column_table_stats.q,\
column_table_stats_orc.q,\
+ colstats_date_min_max.q,\
compare_double_bigint_2.q,\
constprog_dpp.q,\
current_date_timestamp.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/b7ec52e3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 345595b..c59128d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -100,6 +100,8 @@ import
org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -751,8 +753,14 @@ public class StatsRulesProcFactory {
}
} else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) ||
colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
+int value;
+if (colTypeLowerCase == serdeConstants.DATE_TYPE_NAME) {
+ DateWritable writableVal = new
DateWritable(java.sql.Date.valueOf(boundValue));
+ value = writableVal.getDays();
+} else {
+ value = new Integer(boundValue);
+}
// Date is an integer internally
-int value = new Integer(boundValue);
int maxValue = cs.getRange().maxValue.intValue();
int minValue = cs.getRange().minValue.intValue();
if (upperBound) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b7ec52e3/ql/src/test/queries/clientpositive/colstats_date_min_max.q
--
diff --git a/ql/src/test/queries/clientpositive/colstats_date_min_max.q
b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
new file mode 100644
index 000..7f5be6a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
@@ -0,0 +1,30 @@
+set hive.explain.user=true;
+
+create table d1(d date);
+-- tblproperties('transactional'='false');
+
+insert into d1 values
+ ('2010-10-01'),
+ ('2010-10-02'),
+ ('2010-10-03'),
+ ('2010-10-04'),
+ ('2010-10-05'),
+ ('2010-10-06'),
+ ('2010-10-07'),
+ ('2010-10-08'),
+ ('2010-10-09'),
+ ('2010-10-10');
+
+analyze table d1 compute statistics for columns;
+
+desc formatted d1;
+desc formatted d1 d;
+
+explain
+select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01';
+
+explain
+s
[2/2] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19500: Prevent multiple selectivity estimations for the same variable in
conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fdb8a70c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fdb8a70c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fdb8a70c
Branch: refs/heads/master
Commit: fdb8a70cd548b702801c8abc3464604c04addf63
Parents: 5502537
Author: Zoltan Haindrich
Authored: Fri May 25 14:07:04 2018 +0200
Committer: Zoltan Haindrich
Committed: Fri May 25 14:07:04 2018 +0200
--
.../stats/annotation/StatsRulesProcFactory.java | 12 +-
.../clientpositive/stat_estimate_related_col.q | 43 +
.../annotate_stats_deep_filters.q.out | 4 +-
.../annotate_stats_join_pkfk.q.out | 48 +-
.../llap/bucketsortoptimize_insert_7.q.out | 20 +-
.../clientpositive/llap/explainuser_1.q.out | 64 +-
.../llap/orc_predicate_pushdown.q.out | 20 +-
.../llap/parquet_predicate_pushdown.q.out | 20 +-
.../clientpositive/llap/subquery_multi.q.out| 26 +-
.../llap/vector_decimal_cast.q.out | 4 +-
.../clientpositive/llap/vector_if_expr.q.out| 10 +-
.../llap/vector_mapjoin_reduce.q.out| 14 +-
.../clientpositive/llap/vectorization_0.q.out | 12 +-
.../llap/vectorization_short_regress.q.out | 8 +-
.../llap/vectorized_nested_mapjoin.q.out| 12 +-
.../spark/spark_explainuser_1.q.out | 68 +-
.../stat_estimate_related_col.q.out | 784 +++
17 files changed, 998 insertions(+), 171 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/fdb8a70c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index c59128d..91cccfb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -349,12 +349,12 @@ public class StatsRulesProcFactory {
for (ExprNodeDesc child : genFunc.getChildren()) {
evaluatedRowCount = evaluateChildExpr(aspCtx.getAndExprStats(),
child,
aspCtx, neededCols, op, evaluatedRowCount);
- }
- newNumRows = evaluatedRowCount;
- if (satisfyPrecondition(aspCtx.getAndExprStats())) {
-updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
- } else {
-updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+newNumRows = evaluatedRowCount;
+if (satisfyPrecondition(aspCtx.getAndExprStats())) {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
+} else {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+}
}
} else if (udf instanceof GenericUDFOPOr) {
// for OR condition independently compute and update stats.
http://git-wip-us.apache.org/repos/asf/hive/blob/fdb8a70c/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
--
diff --git a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
new file mode 100644
index 000..52da2f7
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
@@ -0,0 +1,43 @@
+set hive.explain.user=true;
+set hive.strict.checks.cartesian.product=false;
+set hive.stats.fetch.column.stats=true;
+
+drop table if exists t1;
+drop table if exists t8;
+
+create table t1 (a integer,b integer);
+create table t8 like t1;
+
+insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+insert into t8
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1 union all
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1
+;
+
+analyze table t1 compute statistics for columns;
+analyze table t8 compute statistics for columns;
+
+explain analyze select sum(a) from t8 where b in (2,3) group by b;
+explain analyze select sum(a) from t8 where b=2 group by b;
+
+explain analyze select sum(a) from t1 where 2=b and b=2 group by b;
+
+explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b;
+explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b;
+
+
+explain analyze select
[1/2] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master 55025372a -> fdb8a70cd http://git-wip-us.apache.org/repos/asf/hive/blob/fdb8a70c/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out -- diff --git a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out new file mode 100644 index 000..8cc0fbc --- /dev/null +++ b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out @@ -0,0 +1,784 @@ +PREHOOK: query: drop table if exists t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists t8 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t8 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 (a integer,b integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 (a integer,b integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t8 like t1 +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8 like t1 +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1 +POSTHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.a SCRIPT [] +POSTHOOK: Lineage: t1.b SCRIPT [] +PREHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t8 +POSTHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t8.a EXPRESSION [(t1)t1.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: t8.b EXPRESSION [(t1)t1.FieldSchema(name:b, type:int, comment:null), ] +PREHOOK: query: analyze table t1 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 + A masked pattern was here +POSTHOOK: query: analyze table t1 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 + A masked pattern was here +PREHOOK: query: analyze table t8 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t8 +PREHOOK: Output: default@t8 + A masked pattern was here +POSTHOOK: query: analyze table t8 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t8 +POSTHOOK: Output: default@t8 + A masked pattern was here +PREHOOK: query: select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +PREHOOK: Input: default@t8 + A masked pattern was here +POSTHOOK: query: select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8 + A masked pattern was here +PREHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: t8 +Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE +Filter Operator + predicate: (b) IN (2, 3) (type: boolean) + Statistics: Num rows: 16/16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator +aggregations: sum(a) +keys: b (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLET
hive git commit: HIVE-19557: stats: filters for dates are not taking advantage of min/max values (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master c4c53a646 -> a6832e605
HIVE-19557: stats: filters for dates are not taking advantage of min/max values
(Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6832e60
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6832e60
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6832e60
Branch: refs/heads/master
Commit: a6832e60526585ba859ae0ce38c14ea9014a0bb5
Parents: c4c53a6
Author: Zoltan Haindrich
Authored: Fri May 25 07:00:19 2018 +0200
Committer: Zoltan Haindrich
Committed: Fri May 25 07:00:19 2018 +0200
--
.../test/resources/testconfiguration.properties | 1 +
.../stats/annotation/StatsRulesProcFactory.java | 10 +-
.../clientpositive/colstats_date_min_max.q | 30 +++
.../llap/colstats_date_min_max.q.out| 193 +++
.../clientpositive/llap/vector_between_in.q.out | 20 +-
5 files changed, 243 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a6832e60/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index dcdb0fa..d146f92 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -489,6 +489,7 @@ minillaplocal.query.files=\
cbo_subq_not_in.q,\
column_table_stats.q,\
column_table_stats_orc.q,\
+ colstats_date_min_max.q,\
compare_double_bigint_2.q,\
constprog_dpp.q,\
current_date_timestamp.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/a6832e60/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 345595b..c59128d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -100,6 +100,8 @@ import
org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -751,8 +753,14 @@ public class StatsRulesProcFactory {
}
} else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) ||
colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
+int value;
+if (colTypeLowerCase == serdeConstants.DATE_TYPE_NAME) {
+ DateWritable writableVal = new
DateWritable(java.sql.Date.valueOf(boundValue));
+ value = writableVal.getDays();
+} else {
+ value = new Integer(boundValue);
+}
// Date is an integer internally
-int value = new Integer(boundValue);
int maxValue = cs.getRange().maxValue.intValue();
int minValue = cs.getRange().minValue.intValue();
if (upperBound) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a6832e60/ql/src/test/queries/clientpositive/colstats_date_min_max.q
--
diff --git a/ql/src/test/queries/clientpositive/colstats_date_min_max.q
b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
new file mode 100644
index 000..7f5be6a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
@@ -0,0 +1,30 @@
+set hive.explain.user=true;
+
+create table d1(d date);
+-- tblproperties('transactional'='false');
+
+insert into d1 values
+ ('2010-10-01'),
+ ('2010-10-02'),
+ ('2010-10-03'),
+ ('2010-10-04'),
+ ('2010-10-05'),
+ ('2010-10-06'),
+ ('2010-10-07'),
+ ('2010-10-08'),
+ ('2010-10-09'),
+ ('2010-10-10');
+
+analyze table d1 compute statistics for columns;
+
+desc formatted d1;
+desc formatted d1 d;
+
+explain
+select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01';
+
+explain
+selec
hive git commit: HIVE-19557: stats: filters for dates are not taking advantage of min/max values (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master f34149ab5 -> 0b2d364aa
HIVE-19557: stats: filters for dates are not taking advantage of min/max values
(Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0b2d364a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0b2d364a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0b2d364a
Branch: refs/heads/master
Commit: 0b2d364aa5977c7a15a4b5082cf9ef431dcc394e
Parents: f34149a
Author: Zoltan Haindrich
Authored: Wed May 23 17:15:39 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed May 23 17:15:39 2018 +0200
--
.../test/resources/testconfiguration.properties | 1 +
.../stats/annotation/StatsRulesProcFactory.java | 10 +-
.../clientpositive/colstats_date_min_max.q | 30 +++
.../llap/colstats_date_min_max.q.out| 193 +++
.../clientpositive/llap/vector_between_in.q.out | 20 +-
5 files changed, 243 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/0b2d364a/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 6528ec6..6007d5a 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -488,6 +488,7 @@ minillaplocal.query.files=\
cbo_subq_not_in.q,\
column_table_stats.q,\
column_table_stats_orc.q,\
+ colstats_date_min_max.q,\
compare_double_bigint_2.q,\
constprog_dpp.q,\
current_date_timestamp.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/0b2d364a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index c770227..91cccfb 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -100,6 +100,8 @@ import
org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -751,8 +753,14 @@ public class StatsRulesProcFactory {
}
} else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) ||
colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
+int value;
+if (colTypeLowerCase == serdeConstants.DATE_TYPE_NAME) {
+ DateWritable writableVal = new
DateWritable(java.sql.Date.valueOf(boundValue));
+ value = writableVal.getDays();
+} else {
+ value = new Integer(boundValue);
+}
// Date is an integer internally
-int value = new Integer(boundValue);
int maxValue = cs.getRange().maxValue.intValue();
int minValue = cs.getRange().minValue.intValue();
if (upperBound) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0b2d364a/ql/src/test/queries/clientpositive/colstats_date_min_max.q
--
diff --git a/ql/src/test/queries/clientpositive/colstats_date_min_max.q
b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
new file mode 100644
index 000..7f5be6a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
@@ -0,0 +1,30 @@
+set hive.explain.user=true;
+
+create table d1(d date);
+-- tblproperties('transactional'='false');
+
+insert into d1 values
+ ('2010-10-01'),
+ ('2010-10-02'),
+ ('2010-10-03'),
+ ('2010-10-04'),
+ ('2010-10-05'),
+ ('2010-10-06'),
+ ('2010-10-07'),
+ ('2010-10-08'),
+ ('2010-10-09'),
+ ('2010-10-10');
+
+analyze table d1 compute statistics for columns;
+
+desc formatted d1;
+desc formatted d1 d;
+
+explain
+select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01';
+
+explain
+selec
hive git commit: HIVE-19651: only add arrow dependency once (Igor Kryvenko via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master 24d70279e -> f34149ab5
HIVE-19651: only add arrow dependency once (Igor Kryvenko via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f34149ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f34149ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f34149ab
Branch: refs/heads/master
Commit: f34149ab5de8808fd224cdd2543bcc8d42182b84
Parents: 24d7027
Author: Igor Kryvenko
Authored: Wed May 23 16:53:18 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed May 23 16:53:18 2018 +0200
--
serde/pom.xml | 5 -
1 file changed, 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f34149ab/serde/pom.xml
--
diff --git a/serde/pom.xml b/serde/pom.xml
index 230e0fc..b07c792 100644
--- a/serde/pom.xml
+++ b/serde/pom.xml
@@ -71,11 +71,6 @@
${arrow.version}
- org.apache.arrow
- arrow-vector
- ${arrow.version}
-
-
com.carrotsearch
hppc
${hppc.version}
[1/2] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master 4f789cc6f -> 24d70279e http://git-wip-us.apache.org/repos/asf/hive/blob/24d70279/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out -- diff --git a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out new file mode 100644 index 000..8cc0fbc --- /dev/null +++ b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out @@ -0,0 +1,784 @@ +PREHOOK: query: drop table if exists t1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists t8 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t8 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table t1 (a integer,b integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: create table t1 (a integer,b integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: create table t8 like t1 +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t8 +POSTHOOK: query: create table t8 like t1 +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t8 +PREHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t1 +POSTHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t1 +POSTHOOK: Lineage: t1.a SCRIPT [] +POSTHOOK: Lineage: t1.b SCRIPT [] +PREHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t8 +POSTHOOK: query: insert into t8 +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all +select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t8 +POSTHOOK: Lineage: t8.a EXPRESSION [(t1)t1.FieldSchema(name:a, type:int, comment:null), ] +POSTHOOK: Lineage: t8.b EXPRESSION [(t1)t1.FieldSchema(name:b, type:int, comment:null), ] +PREHOOK: query: analyze table t1 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 + A masked pattern was here +POSTHOOK: query: analyze table t1 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 + A masked pattern was here +PREHOOK: query: analyze table t8 compute statistics for columns +PREHOOK: type: ANALYZE_TABLE +PREHOOK: Input: default@t8 +PREHOOK: Output: default@t8 + A masked pattern was here +POSTHOOK: query: analyze table t8 compute statistics for columns +POSTHOOK: type: ANALYZE_TABLE +POSTHOOK: Input: default@t8 +POSTHOOK: Output: default@t8 + A masked pattern was here +PREHOOK: query: select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +PREHOOK: Input: default@t8 + A masked pattern was here +POSTHOOK: query: select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t8 + A masked pattern was here +PREHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +PREHOOK: type: QUERY +POSTHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: t8 +Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE +Filter Operator + predicate: (b) IN (2, 3) (type: boolean) + Statistics: Num rows: 16/16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator +aggregations: sum(a) +keys: b (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLET
[2/2] hive git commit: HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19500: Prevent multiple selectivity estimations for the same variable in
conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24d70279
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24d70279
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24d70279
Branch: refs/heads/master
Commit: 24d70279ee332fed24272d19555c8c19887230b2
Parents: 4f789cc
Author: Zoltan Haindrich
Authored: Wed May 23 16:47:30 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed May 23 16:47:30 2018 +0200
--
.../stats/annotation/StatsRulesProcFactory.java | 12 +-
.../clientpositive/stat_estimate_related_col.q | 43 +
.../annotate_stats_deep_filters.q.out | 4 +-
.../annotate_stats_join_pkfk.q.out | 48 +-
.../llap/bucketsortoptimize_insert_7.q.out | 20 +-
.../clientpositive/llap/explainuser_1.q.out | 64 +-
.../llap/orc_predicate_pushdown.q.out | 20 +-
.../llap/parquet_predicate_pushdown.q.out | 20 +-
.../clientpositive/llap/subquery_multi.q.out| 26 +-
.../llap/vector_decimal_cast.q.out | 4 +-
.../clientpositive/llap/vector_if_expr.q.out| 10 +-
.../llap/vector_mapjoin_reduce.q.out| 14 +-
.../clientpositive/llap/vectorization_0.q.out | 12 +-
.../llap/vectorization_short_regress.q.out | 8 +-
.../llap/vectorized_nested_mapjoin.q.out| 12 +-
.../spark/spark_explainuser_1.q.out | 68 +-
.../stat_estimate_related_col.q.out | 784 +++
17 files changed, 998 insertions(+), 171 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/24d70279/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 345595b..c770227 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -347,12 +347,12 @@ public class StatsRulesProcFactory {
for (ExprNodeDesc child : genFunc.getChildren()) {
evaluatedRowCount = evaluateChildExpr(aspCtx.getAndExprStats(),
child,
aspCtx, neededCols, op, evaluatedRowCount);
- }
- newNumRows = evaluatedRowCount;
- if (satisfyPrecondition(aspCtx.getAndExprStats())) {
-updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
- } else {
-updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+newNumRows = evaluatedRowCount;
+if (satisfyPrecondition(aspCtx.getAndExprStats())) {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
+} else {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
+}
}
} else if (udf instanceof GenericUDFOPOr) {
// for OR condition independently compute and update stats.
http://git-wip-us.apache.org/repos/asf/hive/blob/24d70279/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
--
diff --git a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
new file mode 100644
index 000..52da2f7
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
@@ -0,0 +1,43 @@
+set hive.explain.user=true;
+set hive.strict.checks.cartesian.product=false;
+set hive.stats.fetch.column.stats=true;
+
+drop table if exists t1;
+drop table if exists t8;
+
+create table t1 (a integer,b integer);
+create table t8 like t1;
+
+insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+insert into t8
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1 union all
+select * from t1 union all select * from t1 union all select * from t1 union
all select * from t1
+;
+
+analyze table t1 compute statistics for columns;
+analyze table t8 compute statistics for columns;
+
+explain analyze select sum(a) from t8 where b in (2,3) group by b;
+explain analyze select sum(a) from t8 where b=2 group by b;
+
+explain analyze select sum(a) from t1 where 2=b and b=2 group by b;
+
+explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b;
+explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b;
+
+
+explain analyze select
hive git commit: HIVE-19159: TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/branch-3 685fc9f4c -> 1db0521b5
HIVE-19159: TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan
Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1db0521b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1db0521b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1db0521b
Branch: refs/heads/branch-3
Commit: 1db0521b593fd1e354c75fd719a98d231ded84d6
Parents: 685fc9f
Author: Laszlo Bodor
Authored: Mon May 14 15:46:50 2018 +0200
Committer: Zoltan Haindrich
Committed: Mon May 14 15:46:50 2018 +0200
--
.../apache/hadoop/hive/ql/TestMTQueries.java| 2 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java| 93 +++-
2 files changed, 50 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/1db0521b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
index 4838856..6ed872d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
@@ -42,7 +42,7 @@ public class TestMTQueries extends BaseTestQueries {
// derby fails creating multiple stats aggregator concurrently
util.getConf().setBoolean("hive.exec.submitviachild", true);
util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
- util.getConf().setBoolean("hive.vectorized.execution.enabled", false);
+ util.getConf().setBoolean("hive.vectorized.execution.enabled", true);
util.getConf().set("hive.stats.dbclass", "fs");
util.getConf().set("hive.mapred.mode", "nonstrict");
util.getConf().set("hive.stats.column.autogather", "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/1db0521b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
--
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 58680fe..f6729a9 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -36,6 +36,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
@@ -1200,11 +1201,13 @@ public class QTestUtil {
DatasetCollection datasets = parser.getDatasets();
for (String table : datasets.getTables()){
- initDataset(table);
+ synchronized (QTestUtil.class){
+initDataset(table);
+ }
}
}
- private void initDataset(String table) {
+ protected void initDataset(String table) {
if (getSrcTables().contains(table)){
return;
}
@@ -1270,7 +1273,7 @@ public class QTestUtil {
initDataSetForTest(file);
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
-"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap(conf);
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
@@ -1287,6 +1290,30 @@ public class QTestUtil {
}
File outf = new File(logDir, stdoutName);
+
+setSessionOutputs(fileName, ss, outf);
+
+SessionState oldSs = SessionState.get();
+
+boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName);
+restartSessions(canReuseSession, ss, oldSs);
+
+closeSession(oldSs);
+
+SessionState.start(ss);
+
+cliDriver = new CliDriver();
+
+if (fileName.equals("init_file.q")) {
+ ss.initFiles.add(AbstractCliConfig.HIVE_ROOT +
"/data/scripts/test_init_file.sql");
+}
+cliDriver.processInitFiles(ss);
+
+return outf.getAbsolutePath();
+ }
+
+ private void setSessionOutputs(String fileName, CliSessionState ss, File
outf)
+ throws FileNotFoundException, Exception, UnsupportedEncodingException {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (qSortQuerySet.contains(fileName)) {
ss.out = new SortPrintStream(fo, "UTF-8");
@@ -1299,10 +1326,12 @@ public class QTestUtil {
}
ss.err = new CachingPrintStream(fo, true, "UTF-8");
ss.setIsSilent(true);
-SessionState oldSs = SessionState.ge
hive git commit: HIVE-19159: TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master 0a70831c9 -> 411bfdb0b
HIVE-19159: TestMTQueries.testMTQueries1 failure (Laszlo Bodor via Zoltan
Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/411bfdb0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/411bfdb0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/411bfdb0
Branch: refs/heads/master
Commit: 411bfdb0b80a59e7c84a665ea1c864ec28e3fe9b
Parents: 0a70831
Author: Laszlo Bodor
Authored: Mon May 14 15:45:52 2018 +0200
Committer: Zoltan Haindrich
Committed: Mon May 14 15:45:52 2018 +0200
--
.../apache/hadoop/hive/ql/TestMTQueries.java| 2 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java| 93 +++-
2 files changed, 50 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/411bfdb0/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
index 4838856..6ed872d 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
@@ -42,7 +42,7 @@ public class TestMTQueries extends BaseTestQueries {
// derby fails creating multiple stats aggregator concurrently
util.getConf().setBoolean("hive.exec.submitviachild", true);
util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
- util.getConf().setBoolean("hive.vectorized.execution.enabled", false);
+ util.getConf().setBoolean("hive.vectorized.execution.enabled", true);
util.getConf().set("hive.stats.dbclass", "fs");
util.getConf().set("hive.mapred.mode", "nonstrict");
util.getConf().set("hive.stats.column.autogather", "false");
http://git-wip-us.apache.org/repos/asf/hive/blob/411bfdb0/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
--
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 750fc69..16571b3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -36,6 +36,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
@@ -1204,11 +1205,13 @@ public class QTestUtil {
DatasetCollection datasets = parser.getDatasets();
for (String table : datasets.getTables()){
- initDataset(table);
+ synchronized (QTestUtil.class){
+initDataset(table);
+ }
}
}
- protected synchronized void initDataset(String table) {
+ protected void initDataset(String table) {
if (getSrcTables().contains(table)){
return;
}
@@ -1275,7 +1278,7 @@ public class QTestUtil {
initDataSetForTest(file);
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
-"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
Utilities.clearWorkMap(conf);
CliSessionState ss = new CliSessionState(conf);
assert ss != null;
@@ -1292,6 +1295,30 @@ public class QTestUtil {
}
File outf = new File(logDir, stdoutName);
+
+setSessionOutputs(fileName, ss, outf);
+
+SessionState oldSs = SessionState.get();
+
+boolean canReuseSession = !qNoSessionReuseQuerySet.contains(fileName);
+restartSessions(canReuseSession, ss, oldSs);
+
+closeSession(oldSs);
+
+SessionState.start(ss);
+
+cliDriver = new CliDriver();
+
+if (fileName.equals("init_file.q")) {
+ ss.initFiles.add(AbstractCliConfig.HIVE_ROOT +
"/data/scripts/test_init_file.sql");
+}
+cliDriver.processInitFiles(ss);
+
+return outf.getAbsolutePath();
+ }
+
+ private void setSessionOutputs(String fileName, CliSessionState ss, File
outf)
+ throws FileNotFoundException, Exception, UnsupportedEncodingException {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (qSortQuerySet.contains(fileName)) {
ss.out = new SortPrintStream(fo, "UTF-8");
@@ -1304,10 +1331,12 @@ public class QTestUtil {
}
ss.err = new CachingPrintStream(fo, true, "UTF-8");
ss.setIsSilent(true);
-SessionState oldSs = Sess
hive git commit: HIVE-19468: Add Apache license to TestTxnConcatenate (Igor Kryvenko via Zoltan Haindrich)
Repository: hive Updated Branches: refs/heads/master fb7987059 -> 8028ce8a4 HIVE-19468: Add Apache license to TestTxnConcatenate (Igor Kryvenko via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8028ce8a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8028ce8a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8028ce8a Branch: refs/heads/master Commit: 8028ce8a4cf5a03e2998c33e032a511fae770b47 Parents: fb79870 Author: Igor Kryvenko Authored: Fri May 11 14:19:54 2018 +0200 Committer: Zoltan Haindrich Committed: Fri May 11 14:19:54 2018 +0200 -- .../apache/hadoop/hive/ql/TestTxnConcatenate.java | 18 ++ 1 file changed, 18 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8028ce8a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java -- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java index 92bcefe..2663fec 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnConcatenate.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.ql; import org.apache.hadoop.hive.conf.HiveConf;
hive git commit: HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 10699bf14 -> da10aabe5
HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich
reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da10aabe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da10aabe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da10aabe
Branch: refs/heads/master
Commit: da10aabe56edf8fbb26d89d64bedcc4afa84a305
Parents: 10699bf
Author: Zoltan Haindrich
Authored: Wed Apr 25 17:41:57 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed Apr 25 17:41:57 2018 +0200
--
data/conf/llap/hive-site.xml| 2 +-
data/conf/perf-reg/tez/hive-site.xml| 2 +-
.../RuntimeStatsPersistenceCheckerHook.java | 71
.../ql/optimizer/signature/SignatureUtils.java | 2 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java| 6 +-
5 files changed, 77 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/data/conf/llap/hive-site.xml
--
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index 990b473..1507a56 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -163,7 +163,7 @@
hive.exec.post.hooks
- org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
+ org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,
org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook
Post Execute Hook for Tests
http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/data/conf/perf-reg/tez/hive-site.xml
--
diff --git a/data/conf/perf-reg/tez/hive-site.xml
b/data/conf/perf-reg/tez/hive-site.xml
index e11f8f8..78a5481 100644
--- a/data/conf/perf-reg/tez/hive-site.xml
+++ b/data/conf/perf-reg/tez/hive-site.xml
@@ -162,7 +162,7 @@
hive.exec.post.hooks
- org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
+ org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,
org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook
Post Execute Hook for Tests
http://git-wip-us.apache.org/repos/asf/hive/blob/da10aabe/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
new file mode 100644
index 000..b0bdad3
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature;
+import org.apache.hadoop.hive.ql.optimizer.signature.RuntimeStatsPersister;
+import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This hook adds a persistence loop-back ensure that runtime statistics could
be used.
+ */
+public class RuntimeStatsPersistenceCheckerHook implements
ExecuteWithHookContext {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(RuntimeStatsPersistenceCheckerHook.class);
+
+ @Override
+ public void run(HookContext hookContext) throws Exception {
+
+PlanMapper pm = ((PrivateHookContext)
hookContext).getContext().getPlanMapper();
+
+List sigs = pm.getAll(OpTreeSignature.class);
+
+for (OpTreeSignature sig : sigs) {
+ try {
+OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class);
+sig.getSig().proveEquals(sig2.getSig());
+ } catch (Exception e) {
+throw new RuntimeException("while checking the signature of: " +
sig.getS
hive git commit: HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 86f0fcf10 -> 6535dce08
HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich
reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6535dce0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6535dce0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6535dce0
Branch: refs/heads/branch-3
Commit: 6535dce08e6e88961aca8fb48f20e8cdf25ec1c0
Parents: 86f0fcf
Author: Zoltan Haindrich
Authored: Wed Apr 25 17:39:01 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed Apr 25 17:39:01 2018 +0200
--
data/conf/llap/hive-site.xml| 2 +-
data/conf/perf-reg/tez/hive-site.xml| 2 +-
.../RuntimeStatsPersistenceCheckerHook.java | 71
.../ql/optimizer/signature/SignatureUtils.java | 2 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java| 6 +-
5 files changed, 77 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/data/conf/llap/hive-site.xml
--
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index 990b473..1507a56 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -163,7 +163,7 @@
hive.exec.post.hooks
- org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
+ org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,
org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook
Post Execute Hook for Tests
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/data/conf/perf-reg/tez/hive-site.xml
--
diff --git a/data/conf/perf-reg/tez/hive-site.xml
b/data/conf/perf-reg/tez/hive-site.xml
index e11f8f8..78a5481 100644
--- a/data/conf/perf-reg/tez/hive-site.xml
+++ b/data/conf/perf-reg/tez/hive-site.xml
@@ -162,7 +162,7 @@
hive.exec.post.hooks
- org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
+ org.apache.hadoop.hive.ql.hooks.PostExecutePrinter,
org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook
Post Execute Hook for Tests
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
new file mode 100644
index 000..b0bdad3
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature;
+import org.apache.hadoop.hive.ql.optimizer.signature.RuntimeStatsPersister;
+import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This hook adds a persistence loop-back ensure that runtime statistics could
be used.
+ */
+public class RuntimeStatsPersistenceCheckerHook implements
ExecuteWithHookContext {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(RuntimeStatsPersistenceCheckerHook.class);
+
+ @Override
+ public void run(HookContext hookContext) throws Exception {
+
+PlanMapper pm = ((PrivateHookContext)
hookContext).getContext().getPlanMapper();
+
+List sigs = pm.getAll(OpTreeSignature.class);
+
+for (OpTreeSignature sig : sigs) {
+ try {
+OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class);
+sig.getSig().proveEquals(sig2.getSig());
+ } catch (Exception e) {
+throw new RuntimeException("while checking the signature of: " +
sig.
[2/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index 9c94942..1c1d58e 100644
---
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -1528,6 +1528,17 @@ interface ThriftHiveMetastoreIf extends
\FacebookServiceIf {
* @return bool
*/
public function heartbeat_lock_materialization_rebuild($dbName, $tableName,
$txnId);
+ /**
+ * @param \metastore\RuntimeStat $stat
+ * @throws \metastore\MetaException
+ */
+ public function add_runtime_stats(\metastore\RuntimeStat $stat);
+ /**
+ * @param \metastore\GetRuntimeStatsRequest $rqst
+ * @return \metastore\RuntimeStat[]
+ * @throws \metastore\MetaException
+ */
+ public function get_runtime_stats(\metastore\GetRuntimeStatsRequest $rqst);
}
class ThriftHiveMetastoreClient extends \FacebookServiceClient implements
\metastore\ThriftHiveMetastoreIf {
@@ -13007,6 +13018,111 @@ class ThriftHiveMetastoreClient extends
\FacebookServiceClient implements \metas
throw new \Exception("heartbeat_lock_materialization_rebuild failed:
unknown result");
}
+ public function add_runtime_stats(\metastore\RuntimeStat $stat)
+ {
+$this->send_add_runtime_stats($stat);
+$this->recv_add_runtime_stats();
+ }
+
+ public function send_add_runtime_stats(\metastore\RuntimeStat $stat)
+ {
+$args = new \metastore\ThriftHiveMetastore_add_runtime_stats_args();
+$args->stat = $stat;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+ thrift_protocol_write_binary($this->output_, 'add_runtime_stats',
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+ $this->output_->writeMessageBegin('add_runtime_stats',
TMessageType::CALL, $this->seqid_);
+ $args->write($this->output_);
+ $this->output_->writeMessageEnd();
+ $this->output_->getTransport()->flush();
+}
+ }
+
+ public function recv_add_runtime_stats()
+ {
+$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_read_binary');
+if ($bin_accel) $result = thrift_protocol_read_binary($this->input_,
'\metastore\ThriftHiveMetastore_add_runtime_stats_result',
$this->input_->isStrictRead());
+else
+{
+ $rseqid = 0;
+ $fname = null;
+ $mtype = 0;
+
+ $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+ if ($mtype == TMessageType::EXCEPTION) {
+$x = new TApplicationException();
+$x->read($this->input_);
+$this->input_->readMessageEnd();
+throw $x;
+ }
+ $result = new \metastore\ThriftHiveMetastore_add_runtime_stats_result();
+ $result->read($this->input_);
+ $this->input_->readMessageEnd();
+}
+if ($result->o1 !== null) {
+ throw $result->o1;
+}
+return;
+ }
+
+ public function get_runtime_stats(\metastore\GetRuntimeStatsRequest $rqst)
+ {
+$this->send_get_runtime_stats($rqst);
+return $this->recv_get_runtime_stats();
+ }
+
+ public function send_get_runtime_stats(\metastore\GetRuntimeStatsRequest
$rqst)
+ {
+$args = new \metastore\ThriftHiveMetastore_get_runtime_stats_args();
+$args->rqst = $rqst;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+ thrift_protocol_write_binary($this->output_, 'get_runtime_stats',
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+ $this->output_->writeMessageBegin('get_runtime_stats',
TMessageType::CALL, $this->seqid_);
+ $args->write($this->output_);
+ $this->output_->writeMessageEnd();
+ $this->output_->getTransport()->flush();
+}
+ }
+
+ public function recv_get_runtime_stats()
+ {
+$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_read_binary');
+if ($bin_accel) $result = thrift_protocol_read_binary($this->input_,
'\metastore\ThriftHiveMetastore_get_runtime_stats_result',
$this->input_->isStrictRead());
+else
+{
+ $rseqid = 0;
+ $fname = null;
+ $mtype = 0;
+
+ $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+ if ($mtype == TMessageType::EXCEPTION) {
+$x = new TApplicationException();
+$x->read($this->input_);
+$this->input_->readMessageEnd();
+throw $x;
+ }
+ $result = new \metastore\ThriftHiveMetastore_get_ru
[4/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
index 802d8e3..da66951 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
@@ -224,6 +224,8 @@ class ThriftHiveMetastoreIf : virtual public
::facebook::fb303::FacebookService
virtual void get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst) = 0;
virtual void get_lock_materialization_rebuild(LockResponse& _return, const
std::string& dbName, const std::string& tableName, const int64_t txnId) = 0;
virtual bool heartbeat_lock_materialization_rebuild(const std::string&
dbName, const std::string& tableName, const int64_t txnId) = 0;
+ virtual void add_runtime_stats(const RuntimeStat& stat) = 0;
+ virtual void get_runtime_stats(std::vector & _return, const
GetRuntimeStatsRequest& rqst) = 0;
};
class ThriftHiveMetastoreIfFactory : virtual public
::facebook::fb303::FacebookServiceIfFactory {
@@ -887,6 +889,12 @@ class ThriftHiveMetastoreNull : virtual public
ThriftHiveMetastoreIf , virtual p
bool _return = false;
return _return;
}
+ void add_runtime_stats(const RuntimeStat& /* stat */) {
+return;
+ }
+ void get_runtime_stats(std::vector & /* _return */, const
GetRuntimeStatsRequest& /* rqst */) {
+return;
+ }
};
typedef struct _ThriftHiveMetastore_getMetaConf_args__isset {
@@ -25644,6 +25652,222 @@ class
ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult {
};
+typedef struct _ThriftHiveMetastore_add_runtime_stats_args__isset {
+ _ThriftHiveMetastore_add_runtime_stats_args__isset() : stat(false) {}
+ bool stat :1;
+} _ThriftHiveMetastore_add_runtime_stats_args__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_args {
+ public:
+
+ ThriftHiveMetastore_add_runtime_stats_args(const
ThriftHiveMetastore_add_runtime_stats_args&);
+ ThriftHiveMetastore_add_runtime_stats_args& operator=(const
ThriftHiveMetastore_add_runtime_stats_args&);
+ ThriftHiveMetastore_add_runtime_stats_args() {
+ }
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_args() throw();
+ RuntimeStat stat;
+
+ _ThriftHiveMetastore_add_runtime_stats_args__isset __isset;
+
+ void __set_stat(const RuntimeStat& val);
+
+ bool operator == (const ThriftHiveMetastore_add_runtime_stats_args & rhs)
const
+ {
+if (!(stat == rhs.stat))
+ return false;
+return true;
+ }
+ bool operator != (const ThriftHiveMetastore_add_runtime_stats_args &rhs)
const {
+return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_add_runtime_stats_args & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+
+class ThriftHiveMetastore_add_runtime_stats_pargs {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_pargs() throw();
+ const RuntimeStat* stat;
+
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_add_runtime_stats_result__isset {
+ _ThriftHiveMetastore_add_runtime_stats_result__isset() : o1(false) {}
+ bool o1 :1;
+} _ThriftHiveMetastore_add_runtime_stats_result__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_result {
+ public:
+
+ ThriftHiveMetastore_add_runtime_stats_result(const
ThriftHiveMetastore_add_runtime_stats_result&);
+ ThriftHiveMetastore_add_runtime_stats_result& operator=(const
ThriftHiveMetastore_add_runtime_stats_result&);
+ ThriftHiveMetastore_add_runtime_stats_result() {
+ }
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_result() throw();
+ MetaException o1;
+
+ _ThriftHiveMetastore_add_runtime_stats_result__isset __isset;
+
+ void __set_o1(const MetaException& val);
+
+ bool operator == (const ThriftHiveMetastore_add_runtime_stats_result & rhs)
const
+ {
+if (!(o1 == rhs.o1))
+ return false;
+return true;
+ }
+ bool operator != (const ThriftHiveMetastore_add_runtime_stats_result &rhs)
const {
+return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_add_runtime_stats_result & )
const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_add_runtime_stats_presult__isset {
+ _ThriftHiveMetastore_add_runtime_stats_presult__isset() : o1(false) {}
+ bool o1 :1;
+} _ThriftHiveMetastore_add_runtime_stats_presult__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_presult {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_presult() throw();
+ MetaException o1;
+
+ _ThriftHiveMetastore_add_runtime_stats_presu
[6/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86f0fcf1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86f0fcf1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86f0fcf1 Branch: refs/heads/branch-3 Commit: 86f0fcf10e8d7c62bf75dfb2f7fa620403d73229 Parents: 4db5b55 Author: Zoltan Haindrich Authored: Wed Apr 25 16:39:50 2018 +0200 Committer: Zoltan Haindrich Committed: Wed Apr 25 16:39:50 2018 +0200 -- .../org/apache/hadoop/hive/conf/HiveConf.java |6 +- .../listener/DummyRawStoreFailEvent.java| 19 + .../org/apache/hadoop/hive/ql/QTestUtil.java|2 +- .../upgrade/derby/056-HIVE-19171.derby.sql | 10 + .../ql/optimizer/signature/OpSignature.java | 19 +- .../ql/optimizer/signature/OpTreeSignature.java | 24 +- .../signature/OpTreeSignatureFactory.java | 12 +- .../ql/optimizer/signature/RuntimeStatsMap.java | 83 + .../signature/RuntimeStatsPersister.java| 54 + .../ql/optimizer/signature/SignatureUtils.java | 22 +- .../hadoop/hive/ql/plan/FileSinkDesc.java |7 +- .../hadoop/hive/ql/plan/HashTableSinkDesc.java |6 +- .../apache/hadoop/hive/ql/plan/JoinDesc.java|6 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java |6 +- .../hive/ql/plan/mapper/CachingStatsSource.java |7 +- .../hive/ql/plan/mapper/EmptyStatsSource.java |2 +- .../ql/plan/mapper/MapBackedStatsSource.java| 50 + .../ql/plan/mapper/MetastoreStatsConnector.java | 143 + .../hadoop/hive/ql/plan/mapper/PlanMapper.java | 108 +- .../plan/mapper/SimpleRuntimeStatsSource.java | 65 - .../hive/ql/plan/mapper/StatsSources.java | 86 +- .../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 17 +- .../hadoop/hive/ql/stats/OperatorStats.java | 33 +- .../signature/TestRuntimeStatsPersistence.java | 165 + .../ql/plan/mapping/TestCounterMapping.java |7 +- .../ql/plan/mapping/TestReOptimization.java | 85 +- .../apache/hive/service/server/HiveServer2.java |3 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 5376 ++ .../gen/thrift/gen-cpp/ThriftHiveMetastore.h| 259 + .../ThriftHiveMetastore_server.skeleton.cpp | 10 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 376 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 97 + .../metastore/api/GetRuntimeStatsRequest.java | 283 + .../hadoop/hive/metastore/api/RuntimeStat.java | 600 ++ .../hive/metastore/api/ThriftHiveMetastore.java | 2584 +++-- .../gen-php/metastore/ThriftHiveMetastore.php | 481 ++ .../src/gen/thrift/gen-php/metastore/Types.php | 171 + .../hive_metastore/ThriftHiveMetastore-remote | 14 + .../hive_metastore/ThriftHiveMetastore.py | 409 ++ .../gen/thrift/gen-py/hive_metastore/ttypes.py | 141 + .../gen/thrift/gen-rb/hive_metastore_types.rb | 37 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 119 + .../hadoop/hive/metastore/HiveMetaStore.java| 112 +- .../hive/metastore/HiveMetaStoreClient.java | 136 +- .../hadoop/hive/metastore/IMetaStoreClient.java |9 + .../hadoop/hive/metastore/ObjectStore.java | 248 +- .../apache/hadoop/hive/metastore/RawStore.java | 15 +- .../hive/metastore/RuntimeStatsCleanerTask.java | 67 + .../hive/metastore/cache/CachedStore.java | 21 +- .../hive/metastore/conf/MetastoreConf.java | 34 +- .../hive/metastore/model/MRuntimeStat.java | 59 + .../src/main/resources/package.jdo | 14 + .../main/sql/derby/hive-schema-3.0.0.derby.sql | 10 + .../sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 10 + .../main/sql/mssql/hive-schema-3.0.0.mssql.sql |9 + .../sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql |9 + .../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 10 + .../sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql |9 + .../sql/oracle/hive-schema-3.0.0.oracle.sql | 10 + .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql|9 + .../sql/postgres/hive-schema-3.0.0.postgres.sql | 11 + .../upgrade-2.3.0-to-3.0.0.postgres.sql |9 + .../src/main/thrift/hive_metastore.thrift | 12 + .../DummyRawStoreControlledCommit.java | 18 + .../DummyRawStoreForJdoConnection.java | 17 + .../HiveMetaStoreClientPreCatalog.java | 10 + .../hive/metastore/client/TestRuntimeStats.java | 100 + 67 files changed, 9972 insertions(+), 3000 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/
[3/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index a354f27..e2f0e82 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -446,6 +446,10 @@ import org.slf4j.LoggerFactory;
public boolean heartbeat_lock_materialization_rebuild(String dbName,
String tableName, long txnId) throws org.apache.thrift.TException;
+public void add_runtime_stats(RuntimeStat stat) throws MetaException,
org.apache.thrift.TException;
+
+public List get_runtime_stats(GetRuntimeStatsRequest rqst)
throws MetaException, org.apache.thrift.TException;
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public interface
AsyncIface extends com.facebook.fb303.FacebookService .AsyncIface {
@@ -854,6 +858,10 @@ import org.slf4j.LoggerFactory;
public void heartbeat_lock_materialization_rebuild(String dbName, String
tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback
resultHandler) throws org.apache.thrift.TException;
+public void add_runtime_stats(RuntimeStat stat,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
+
+public void get_runtime_stats(GetRuntimeStatsRequest rqst,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
Client extends com.facebook.fb303.FacebookService.Client implements Iface {
@@ -6692,6 +6700,55 @@ import org.slf4j.LoggerFactory;
throw new
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
"heartbeat_lock_materialization_rebuild failed: unknown result");
}
+public void add_runtime_stats(RuntimeStat stat) throws MetaException,
org.apache.thrift.TException
+{
+ send_add_runtime_stats(stat);
+ recv_add_runtime_stats();
+}
+
+public void send_add_runtime_stats(RuntimeStat stat) throws
org.apache.thrift.TException
+{
+ add_runtime_stats_args args = new add_runtime_stats_args();
+ args.setStat(stat);
+ sendBase("add_runtime_stats", args);
+}
+
+public void recv_add_runtime_stats() throws MetaException,
org.apache.thrift.TException
+{
+ add_runtime_stats_result result = new add_runtime_stats_result();
+ receiveBase(result, "add_runtime_stats");
+ if (result.o1 != null) {
+throw result.o1;
+ }
+ return;
+}
+
+public List get_runtime_stats(GetRuntimeStatsRequest rqst)
throws MetaException, org.apache.thrift.TException
+{
+ send_get_runtime_stats(rqst);
+ return recv_get_runtime_stats();
+}
+
+public void send_get_runtime_stats(GetRuntimeStatsRequest rqst) throws
org.apache.thrift.TException
+{
+ get_runtime_stats_args args = new get_runtime_stats_args();
+ args.setRqst(rqst);
+ sendBase("get_runtime_stats", args);
+}
+
+public List recv_get_runtime_stats() throws MetaException,
org.apache.thrift.TException
+{
+ get_runtime_stats_result result = new get_runtime_stats_result();
+ receiveBase(result, "get_runtime_stats");
+ if (result.isSetSuccess()) {
+return result.success;
+ }
+ if (result.o1 != null) {
+throw result.o1;
+ }
+ throw new
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
"get_runtime_stats failed: unknown result");
+}
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
AsyncClient extends com.facebook.fb303.FacebookService.AsyncClient implements
AsyncIface {
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
Factory implements org.apache.thrift.async.TAsyncClientFactory {
@@ -13660,6 +13717,70 @@ import org.slf4j.LoggerFactory;
}
}
+public void add_runtime_stats(RuntimeStat stat,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException {
+ checkReady();
+ add_runtime_stats_call method_call = new add_runtime_stats_call(stat,
resultHandler, this, ___proto
[5/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index dfa13a0..4787703 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -2107,14 +2107,14 @@ uint32_t
ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size1187;
-::apache::thrift::protocol::TType _etype1190;
-xfer += iprot->readListBegin(_etype1190, _size1187);
-this->success.resize(_size1187);
-uint32_t _i1191;
-for (_i1191 = 0; _i1191 < _size1187; ++_i1191)
+uint32_t _size1191;
+::apache::thrift::protocol::TType _etype1194;
+xfer += iprot->readListBegin(_etype1194, _size1191);
+this->success.resize(_size1191);
+uint32_t _i1195;
+for (_i1195 = 0; _i1195 < _size1191; ++_i1195)
{
- xfer += iprot->readString(this->success[_i1191]);
+ xfer += iprot->readString(this->success[_i1195]);
}
xfer += iprot->readListEnd();
}
@@ -2153,10 +2153,10 @@ uint32_t
ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1192;
- for (_iter1192 = this->success.begin(); _iter1192 !=
this->success.end(); ++_iter1192)
+ std::vector ::const_iterator _iter1196;
+ for (_iter1196 = this->success.begin(); _iter1196 !=
this->success.end(); ++_iter1196)
{
-xfer += oprot->writeString((*_iter1192));
+xfer += oprot->writeString((*_iter1196));
}
xfer += oprot->writeListEnd();
}
@@ -2201,14 +2201,14 @@ uint32_t
ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
-uint32_t _size1193;
-::apache::thrift::protocol::TType _etype1196;
-xfer += iprot->readListBegin(_etype1196, _size1193);
-(*(this->success)).resize(_size1193);
-uint32_t _i1197;
-for (_i1197 = 0; _i1197 < _size1193; ++_i1197)
+uint32_t _size1197;
+::apache::thrift::protocol::TType _etype1200;
+xfer += iprot->readListBegin(_etype1200, _size1197);
+(*(this->success)).resize(_size1197);
+uint32_t _i1201;
+for (_i1201 = 0; _i1201 < _size1197; ++_i1201)
{
- xfer += iprot->readString((*(this->success))[_i1197]);
+ xfer += iprot->readString((*(this->success))[_i1201]);
}
xfer += iprot->readListEnd();
}
@@ -2325,14 +2325,14 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size1198;
-::apache::thrift::protocol::TType _etype1201;
-xfer += iprot->readListBegin(_etype1201, _size1198);
-this->success.resize(_size1198);
-uint32_t _i1202;
-for (_i1202 = 0; _i1202 < _size1198; ++_i1202)
+uint32_t _size1202;
+::apache::thrift::protocol::TType _etype1205;
+xfer += iprot->readListBegin(_etype1205, _size1202);
+this->success.resize(_size1202);
+uint32_t _i1206;
+for (_i1206 = 0; _i1206 < _size1202; ++_i1206)
{
- xfer += iprot->readString(this->success[_i1202]);
+ xfer += iprot->readString(this->success[_i1206]);
}
xfer += iprot->readListEnd();
}
@@ -2371,10 +2371,10 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1203;
- for (_iter1203 = this->success.begin(); _iter1203 !=
this->success.end(); ++_iter1203)
+ std::vector ::const_iterator _iter1207;
+ for (_iter1207 = this->success.begin(); _iter1207 !=
this->success.end(); ++_iter1207)
{
-xfer += opro
[1/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 4db5b55a3 -> 86f0fcf10
http://git-wip-us.apache.org/repos/asf/hive/blob/86f0fcf1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 125d5a7..184ecb6 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -55,6 +55,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.jdo.JDOCanRetryException;
import javax.jdo.JDODataStoreException;
@@ -83,7 +84,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.BasicTxnInfo;
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
@@ -124,6 +124,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceType;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.RuntimeStat;
import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
@@ -186,6 +187,7 @@ import
org.apache.hadoop.hive.metastore.model.MPartitionPrivilege;
import org.apache.hadoop.hive.metastore.model.MResourceUri;
import org.apache.hadoop.hive.metastore.model.MRole;
import org.apache.hadoop.hive.metastore.model.MRoleMap;
+import org.apache.hadoop.hive.metastore.model.MRuntimeStat;
import org.apache.hadoop.hive.metastore.model.MSchemaVersion;
import org.apache.hadoop.hive.metastore.model.MSerDeInfo;
import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
@@ -210,7 +212,6 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
-import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.datanucleus.AbstractNucleusContext;
import org.datanucleus.ClassLoaderResolver;
@@ -809,7 +810,9 @@ public class ObjectStore implements RawStore, Configurable {
pm.makePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -832,7 +835,9 @@ public class ObjectStore implements RawStore, Configurable {
pm.makePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -840,7 +845,9 @@ public class ObjectStore implements RawStore, Configurable {
public Catalog getCatalog(String catalogName) throws NoSuchObjectException,
MetaException {
LOG.debug("Fetching catalog " + catalogName);
MCatalog mCat = getMCatalog(catalogName);
-if (mCat == null) throw new NoSuchObjectException("No catalog " +
catalogName);
+if (mCat == null) {
+ throw new NoSuchObjectException("No catalog " + catalogName);
+}
return mCatToCat(mCat);
}
@@ -874,11 +881,15 @@ public class ObjectStore implements RawStore,
Configurable {
openTransaction();
MCatalog mCat = getMCatalog(catalogName);
pm.retrieve(mCat);
- if (mCat == null) throw new NoSuchObjectException("No catalog " +
catalogName);
+ if (mCat == null) {
+throw new NoSuchObjectException("No catalog " + catalogName);
+ }
pm.deletePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -903,14 +914,18 @@ public class ObjectStore implements RawStore,
Configurable {
private MCatalog catToMCat(Catalog cat) {
MCatalog mCat = new MCatalog();
mCat.setName(normalizeIdentifier(cat.getName()));
-if (cat.isSetDescription()) mCat.setDescription(cat.getDescription());
+if (cat.isSetDescription()) {
+
[5/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index dfa13a0..4787703 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -2107,14 +2107,14 @@ uint32_t
ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size1187;
-::apache::thrift::protocol::TType _etype1190;
-xfer += iprot->readListBegin(_etype1190, _size1187);
-this->success.resize(_size1187);
-uint32_t _i1191;
-for (_i1191 = 0; _i1191 < _size1187; ++_i1191)
+uint32_t _size1191;
+::apache::thrift::protocol::TType _etype1194;
+xfer += iprot->readListBegin(_etype1194, _size1191);
+this->success.resize(_size1191);
+uint32_t _i1195;
+for (_i1195 = 0; _i1195 < _size1191; ++_i1195)
{
- xfer += iprot->readString(this->success[_i1191]);
+ xfer += iprot->readString(this->success[_i1195]);
}
xfer += iprot->readListEnd();
}
@@ -2153,10 +2153,10 @@ uint32_t
ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1192;
- for (_iter1192 = this->success.begin(); _iter1192 !=
this->success.end(); ++_iter1192)
+ std::vector ::const_iterator _iter1196;
+ for (_iter1196 = this->success.begin(); _iter1196 !=
this->success.end(); ++_iter1196)
{
-xfer += oprot->writeString((*_iter1192));
+xfer += oprot->writeString((*_iter1196));
}
xfer += oprot->writeListEnd();
}
@@ -2201,14 +2201,14 @@ uint32_t
ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
-uint32_t _size1193;
-::apache::thrift::protocol::TType _etype1196;
-xfer += iprot->readListBegin(_etype1196, _size1193);
-(*(this->success)).resize(_size1193);
-uint32_t _i1197;
-for (_i1197 = 0; _i1197 < _size1193; ++_i1197)
+uint32_t _size1197;
+::apache::thrift::protocol::TType _etype1200;
+xfer += iprot->readListBegin(_etype1200, _size1197);
+(*(this->success)).resize(_size1197);
+uint32_t _i1201;
+for (_i1201 = 0; _i1201 < _size1197; ++_i1201)
{
- xfer += iprot->readString((*(this->success))[_i1197]);
+ xfer += iprot->readString((*(this->success))[_i1201]);
}
xfer += iprot->readListEnd();
}
@@ -2325,14 +2325,14 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size1198;
-::apache::thrift::protocol::TType _etype1201;
-xfer += iprot->readListBegin(_etype1201, _size1198);
-this->success.resize(_size1198);
-uint32_t _i1202;
-for (_i1202 = 0; _i1202 < _size1198; ++_i1202)
+uint32_t _size1202;
+::apache::thrift::protocol::TType _etype1205;
+xfer += iprot->readListBegin(_etype1205, _size1202);
+this->success.resize(_size1202);
+uint32_t _i1206;
+for (_i1206 = 0; _i1206 < _size1202; ++_i1206)
{
- xfer += iprot->readString(this->success[_i1202]);
+ xfer += iprot->readString(this->success[_i1206]);
}
xfer += iprot->readListEnd();
}
@@ -2371,10 +2371,10 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1203;
- for (_iter1203 = this->success.begin(); _iter1203 !=
this->success.end(); ++_iter1203)
+ std::vector ::const_iterator _iter1207;
+ for (_iter1207 = this->success.begin(); _iter1207 !=
this->success.end(); ++_iter1207)
{
-xfer += opro
[3/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index a354f27..e2f0e82 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -446,6 +446,10 @@ import org.slf4j.LoggerFactory;
public boolean heartbeat_lock_materialization_rebuild(String dbName,
String tableName, long txnId) throws org.apache.thrift.TException;
+public void add_runtime_stats(RuntimeStat stat) throws MetaException,
org.apache.thrift.TException;
+
+public List get_runtime_stats(GetRuntimeStatsRequest rqst)
throws MetaException, org.apache.thrift.TException;
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public interface
AsyncIface extends com.facebook.fb303.FacebookService .AsyncIface {
@@ -854,6 +858,10 @@ import org.slf4j.LoggerFactory;
public void heartbeat_lock_materialization_rebuild(String dbName, String
tableName, long txnId, org.apache.thrift.async.AsyncMethodCallback
resultHandler) throws org.apache.thrift.TException;
+public void add_runtime_stats(RuntimeStat stat,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
+
+public void get_runtime_stats(GetRuntimeStatsRequest rqst,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
Client extends com.facebook.fb303.FacebookService.Client implements Iface {
@@ -6692,6 +6700,55 @@ import org.slf4j.LoggerFactory;
throw new
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
"heartbeat_lock_materialization_rebuild failed: unknown result");
}
+public void add_runtime_stats(RuntimeStat stat) throws MetaException,
org.apache.thrift.TException
+{
+ send_add_runtime_stats(stat);
+ recv_add_runtime_stats();
+}
+
+public void send_add_runtime_stats(RuntimeStat stat) throws
org.apache.thrift.TException
+{
+ add_runtime_stats_args args = new add_runtime_stats_args();
+ args.setStat(stat);
+ sendBase("add_runtime_stats", args);
+}
+
+public void recv_add_runtime_stats() throws MetaException,
org.apache.thrift.TException
+{
+ add_runtime_stats_result result = new add_runtime_stats_result();
+ receiveBase(result, "add_runtime_stats");
+ if (result.o1 != null) {
+throw result.o1;
+ }
+ return;
+}
+
+public List get_runtime_stats(GetRuntimeStatsRequest rqst)
throws MetaException, org.apache.thrift.TException
+{
+ send_get_runtime_stats(rqst);
+ return recv_get_runtime_stats();
+}
+
+public void send_get_runtime_stats(GetRuntimeStatsRequest rqst) throws
org.apache.thrift.TException
+{
+ get_runtime_stats_args args = new get_runtime_stats_args();
+ args.setRqst(rqst);
+ sendBase("get_runtime_stats", args);
+}
+
+public List recv_get_runtime_stats() throws MetaException,
org.apache.thrift.TException
+{
+ get_runtime_stats_result result = new get_runtime_stats_result();
+ receiveBase(result, "get_runtime_stats");
+ if (result.isSetSuccess()) {
+return result.success;
+ }
+ if (result.o1 != null) {
+throw result.o1;
+ }
+ throw new
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
"get_runtime_stats failed: unknown result");
+}
+
}
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
AsyncClient extends com.facebook.fb303.FacebookService.AsyncClient implements
AsyncIface {
@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public static class
Factory implements org.apache.thrift.async.TAsyncClientFactory {
@@ -13660,6 +13717,70 @@ import org.slf4j.LoggerFactory;
}
}
+public void add_runtime_stats(RuntimeStat stat,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException {
+ checkReady();
+ add_runtime_stats_call method_call = new add_runtime_stats_call(stat,
resultHandler, this, ___proto
[2/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index 9c94942..1c1d58e 100644
---
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -1528,6 +1528,17 @@ interface ThriftHiveMetastoreIf extends
\FacebookServiceIf {
* @return bool
*/
public function heartbeat_lock_materialization_rebuild($dbName, $tableName,
$txnId);
+ /**
+ * @param \metastore\RuntimeStat $stat
+ * @throws \metastore\MetaException
+ */
+ public function add_runtime_stats(\metastore\RuntimeStat $stat);
+ /**
+ * @param \metastore\GetRuntimeStatsRequest $rqst
+ * @return \metastore\RuntimeStat[]
+ * @throws \metastore\MetaException
+ */
+ public function get_runtime_stats(\metastore\GetRuntimeStatsRequest $rqst);
}
class ThriftHiveMetastoreClient extends \FacebookServiceClient implements
\metastore\ThriftHiveMetastoreIf {
@@ -13007,6 +13018,111 @@ class ThriftHiveMetastoreClient extends
\FacebookServiceClient implements \metas
throw new \Exception("heartbeat_lock_materialization_rebuild failed:
unknown result");
}
+ public function add_runtime_stats(\metastore\RuntimeStat $stat)
+ {
+$this->send_add_runtime_stats($stat);
+$this->recv_add_runtime_stats();
+ }
+
+ public function send_add_runtime_stats(\metastore\RuntimeStat $stat)
+ {
+$args = new \metastore\ThriftHiveMetastore_add_runtime_stats_args();
+$args->stat = $stat;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+ thrift_protocol_write_binary($this->output_, 'add_runtime_stats',
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+ $this->output_->writeMessageBegin('add_runtime_stats',
TMessageType::CALL, $this->seqid_);
+ $args->write($this->output_);
+ $this->output_->writeMessageEnd();
+ $this->output_->getTransport()->flush();
+}
+ }
+
+ public function recv_add_runtime_stats()
+ {
+$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_read_binary');
+if ($bin_accel) $result = thrift_protocol_read_binary($this->input_,
'\metastore\ThriftHiveMetastore_add_runtime_stats_result',
$this->input_->isStrictRead());
+else
+{
+ $rseqid = 0;
+ $fname = null;
+ $mtype = 0;
+
+ $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+ if ($mtype == TMessageType::EXCEPTION) {
+$x = new TApplicationException();
+$x->read($this->input_);
+$this->input_->readMessageEnd();
+throw $x;
+ }
+ $result = new \metastore\ThriftHiveMetastore_add_runtime_stats_result();
+ $result->read($this->input_);
+ $this->input_->readMessageEnd();
+}
+if ($result->o1 !== null) {
+ throw $result->o1;
+}
+return;
+ }
+
+ public function get_runtime_stats(\metastore\GetRuntimeStatsRequest $rqst)
+ {
+$this->send_get_runtime_stats($rqst);
+return $this->recv_get_runtime_stats();
+ }
+
+ public function send_get_runtime_stats(\metastore\GetRuntimeStatsRequest
$rqst)
+ {
+$args = new \metastore\ThriftHiveMetastore_get_runtime_stats_args();
+$args->rqst = $rqst;
+$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_write_binary');
+if ($bin_accel)
+{
+ thrift_protocol_write_binary($this->output_, 'get_runtime_stats',
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
+}
+else
+{
+ $this->output_->writeMessageBegin('get_runtime_stats',
TMessageType::CALL, $this->seqid_);
+ $args->write($this->output_);
+ $this->output_->writeMessageEnd();
+ $this->output_->getTransport()->flush();
+}
+ }
+
+ public function recv_get_runtime_stats()
+ {
+$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_read_binary');
+if ($bin_accel) $result = thrift_protocol_read_binary($this->input_,
'\metastore\ThriftHiveMetastore_get_runtime_stats_result',
$this->input_->isStrictRead());
+else
+{
+ $rseqid = 0;
+ $fname = null;
+ $mtype = 0;
+
+ $this->input_->readMessageBegin($fname, $mtype, $rseqid);
+ if ($mtype == TMessageType::EXCEPTION) {
+$x = new TApplicationException();
+$x->read($this->input_);
+$this->input_->readMessageEnd();
+throw $x;
+ }
+ $result = new \metastore\ThriftHiveMetastore_get_ru
[1/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master f552e745b -> 56c3a9570
http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 125d5a7..184ecb6 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -55,6 +55,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import javax.jdo.JDOCanRetryException;
import javax.jdo.JDODataStoreException;
@@ -83,7 +84,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
-import org.apache.hadoop.hive.metastore.api.BasicTxnInfo;
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
@@ -124,6 +124,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceType;
import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
+import org.apache.hadoop.hive.metastore.api.RuntimeStat;
import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
@@ -186,6 +187,7 @@ import
org.apache.hadoop.hive.metastore.model.MPartitionPrivilege;
import org.apache.hadoop.hive.metastore.model.MResourceUri;
import org.apache.hadoop.hive.metastore.model.MRole;
import org.apache.hadoop.hive.metastore.model.MRoleMap;
+import org.apache.hadoop.hive.metastore.model.MRuntimeStat;
import org.apache.hadoop.hive.metastore.model.MSchemaVersion;
import org.apache.hadoop.hive.metastore.model.MSerDeInfo;
import org.apache.hadoop.hive.metastore.model.MStorageDescriptor;
@@ -210,7 +212,6 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
-import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.datanucleus.AbstractNucleusContext;
import org.datanucleus.ClassLoaderResolver;
@@ -809,7 +810,9 @@ public class ObjectStore implements RawStore, Configurable {
pm.makePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -832,7 +835,9 @@ public class ObjectStore implements RawStore, Configurable {
pm.makePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -840,7 +845,9 @@ public class ObjectStore implements RawStore, Configurable {
public Catalog getCatalog(String catalogName) throws NoSuchObjectException,
MetaException {
LOG.debug("Fetching catalog " + catalogName);
MCatalog mCat = getMCatalog(catalogName);
-if (mCat == null) throw new NoSuchObjectException("No catalog " +
catalogName);
+if (mCat == null) {
+ throw new NoSuchObjectException("No catalog " + catalogName);
+}
return mCatToCat(mCat);
}
@@ -874,11 +881,15 @@ public class ObjectStore implements RawStore,
Configurable {
openTransaction();
MCatalog mCat = getMCatalog(catalogName);
pm.retrieve(mCat);
- if (mCat == null) throw new NoSuchObjectException("No catalog " +
catalogName);
+ if (mCat == null) {
+throw new NoSuchObjectException("No catalog " + catalogName);
+ }
pm.deletePersistent(mCat);
committed = commitTransaction();
} finally {
- if (!committed) rollbackTransaction();
+ if (!committed) {
+rollbackTransaction();
+ }
}
}
@@ -903,14 +914,18 @@ public class ObjectStore implements RawStore,
Configurable {
private MCatalog catToMCat(Catalog cat) {
MCatalog mCat = new MCatalog();
mCat.setName(normalizeIdentifier(cat.getName()));
-if (cat.isSetDescription()) mCat.setDescription(cat.getDescription());
+if (cat.isSetDescription()) {
+ m
[6/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/56c3a957 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/56c3a957 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/56c3a957 Branch: refs/heads/master Commit: 56c3a95703b9dbb54a0d19b08b4e4a31664b8115 Parents: f552e74 Author: Zoltan Haindrich Authored: Tue Apr 24 07:28:50 2018 +0200 Committer: Zoltan Haindrich Committed: Tue Apr 24 07:28:50 2018 +0200 -- .../org/apache/hadoop/hive/conf/HiveConf.java |6 +- .../listener/DummyRawStoreFailEvent.java| 19 + .../org/apache/hadoop/hive/ql/QTestUtil.java|2 +- .../upgrade/derby/056-HIVE-19171.derby.sql | 10 + .../ql/optimizer/signature/OpSignature.java | 19 +- .../ql/optimizer/signature/OpTreeSignature.java | 24 +- .../signature/OpTreeSignatureFactory.java | 12 +- .../ql/optimizer/signature/RuntimeStatsMap.java | 83 + .../signature/RuntimeStatsPersister.java| 54 + .../ql/optimizer/signature/SignatureUtils.java | 22 +- .../hadoop/hive/ql/plan/FileSinkDesc.java |7 +- .../hadoop/hive/ql/plan/HashTableSinkDesc.java |6 +- .../apache/hadoop/hive/ql/plan/JoinDesc.java|6 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java |6 +- .../hive/ql/plan/mapper/CachingStatsSource.java |7 +- .../hive/ql/plan/mapper/EmptyStatsSource.java |2 +- .../ql/plan/mapper/MapBackedStatsSource.java| 50 + .../ql/plan/mapper/MetastoreStatsConnector.java | 143 + .../hadoop/hive/ql/plan/mapper/PlanMapper.java | 108 +- .../plan/mapper/SimpleRuntimeStatsSource.java | 65 - .../hive/ql/plan/mapper/StatsSources.java | 86 +- .../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 17 +- .../hadoop/hive/ql/stats/OperatorStats.java | 33 +- .../signature/TestRuntimeStatsPersistence.java | 165 + .../ql/plan/mapping/TestCounterMapping.java |7 +- .../ql/plan/mapping/TestReOptimization.java | 85 +- .../apache/hive/service/server/HiveServer2.java |3 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 5376 ++ .../gen/thrift/gen-cpp/ThriftHiveMetastore.h| 259 + .../ThriftHiveMetastore_server.skeleton.cpp | 10 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 376 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 97 + .../metastore/api/GetRuntimeStatsRequest.java | 283 + .../hadoop/hive/metastore/api/RuntimeStat.java | 600 ++ .../hive/metastore/api/ThriftHiveMetastore.java | 2584 +++-- .../gen-php/metastore/ThriftHiveMetastore.php | 481 ++ .../src/gen/thrift/gen-php/metastore/Types.php | 171 + .../hive_metastore/ThriftHiveMetastore-remote | 14 + .../hive_metastore/ThriftHiveMetastore.py | 409 ++ .../gen/thrift/gen-py/hive_metastore/ttypes.py | 141 + .../gen/thrift/gen-rb/hive_metastore_types.rb | 37 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 119 + .../hadoop/hive/metastore/HiveMetaStore.java| 113 +- .../hive/metastore/HiveMetaStoreClient.java | 140 +- .../hadoop/hive/metastore/IMetaStoreClient.java |9 + .../hadoop/hive/metastore/ObjectStore.java | 248 +- .../apache/hadoop/hive/metastore/RawStore.java | 15 +- .../hive/metastore/RuntimeStatsCleanerTask.java | 67 + .../hive/metastore/cache/CachedStore.java | 21 +- .../hive/metastore/conf/MetastoreConf.java | 34 +- .../hive/metastore/model/MRuntimeStat.java | 59 + .../src/main/resources/package.jdo | 14 + .../main/sql/derby/hive-schema-3.0.0.derby.sql | 10 + .../sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 10 + .../main/sql/mssql/hive-schema-3.0.0.mssql.sql |9 + .../sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql |9 + .../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 10 + .../sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql |9 + .../sql/oracle/hive-schema-3.0.0.oracle.sql | 10 + .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql|9 + .../sql/postgres/hive-schema-3.0.0.postgres.sql | 11 + .../upgrade-2.3.0-to-3.0.0.postgres.sql |9 + .../src/main/thrift/hive_metastore.thrift | 12 + .../DummyRawStoreControlledCommit.java | 18 + .../DummyRawStoreForJdoConnection.java | 17 + .../HiveMetaStoreClientPreCatalog.java | 10 + .../hive/metastore/client/TestRuntimeStats.java | 100 + 67 files changed, 9975 insertions(+), 3002 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java -- diff --git a/common/src/java/org/apache/hadoop/hi
[4/6] hive git commit: HIVE-19171: Persist runtime statistics in metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/56c3a957/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
index 802d8e3..da66951 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
@@ -224,6 +224,8 @@ class ThriftHiveMetastoreIf : virtual public
::facebook::fb303::FacebookService
virtual void get_serde(SerDeInfo& _return, const GetSerdeRequest& rqst) = 0;
virtual void get_lock_materialization_rebuild(LockResponse& _return, const
std::string& dbName, const std::string& tableName, const int64_t txnId) = 0;
virtual bool heartbeat_lock_materialization_rebuild(const std::string&
dbName, const std::string& tableName, const int64_t txnId) = 0;
+ virtual void add_runtime_stats(const RuntimeStat& stat) = 0;
+ virtual void get_runtime_stats(std::vector & _return, const
GetRuntimeStatsRequest& rqst) = 0;
};
class ThriftHiveMetastoreIfFactory : virtual public
::facebook::fb303::FacebookServiceIfFactory {
@@ -887,6 +889,12 @@ class ThriftHiveMetastoreNull : virtual public
ThriftHiveMetastoreIf , virtual p
bool _return = false;
return _return;
}
+ void add_runtime_stats(const RuntimeStat& /* stat */) {
+return;
+ }
+ void get_runtime_stats(std::vector & /* _return */, const
GetRuntimeStatsRequest& /* rqst */) {
+return;
+ }
};
typedef struct _ThriftHiveMetastore_getMetaConf_args__isset {
@@ -25644,6 +25652,222 @@ class
ThriftHiveMetastore_heartbeat_lock_materialization_rebuild_presult {
};
+typedef struct _ThriftHiveMetastore_add_runtime_stats_args__isset {
+ _ThriftHiveMetastore_add_runtime_stats_args__isset() : stat(false) {}
+ bool stat :1;
+} _ThriftHiveMetastore_add_runtime_stats_args__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_args {
+ public:
+
+ ThriftHiveMetastore_add_runtime_stats_args(const
ThriftHiveMetastore_add_runtime_stats_args&);
+ ThriftHiveMetastore_add_runtime_stats_args& operator=(const
ThriftHiveMetastore_add_runtime_stats_args&);
+ ThriftHiveMetastore_add_runtime_stats_args() {
+ }
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_args() throw();
+ RuntimeStat stat;
+
+ _ThriftHiveMetastore_add_runtime_stats_args__isset __isset;
+
+ void __set_stat(const RuntimeStat& val);
+
+ bool operator == (const ThriftHiveMetastore_add_runtime_stats_args & rhs)
const
+ {
+if (!(stat == rhs.stat))
+ return false;
+return true;
+ }
+ bool operator != (const ThriftHiveMetastore_add_runtime_stats_args &rhs)
const {
+return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_add_runtime_stats_args & ) const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+
+class ThriftHiveMetastore_add_runtime_stats_pargs {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_pargs() throw();
+ const RuntimeStat* stat;
+
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_add_runtime_stats_result__isset {
+ _ThriftHiveMetastore_add_runtime_stats_result__isset() : o1(false) {}
+ bool o1 :1;
+} _ThriftHiveMetastore_add_runtime_stats_result__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_result {
+ public:
+
+ ThriftHiveMetastore_add_runtime_stats_result(const
ThriftHiveMetastore_add_runtime_stats_result&);
+ ThriftHiveMetastore_add_runtime_stats_result& operator=(const
ThriftHiveMetastore_add_runtime_stats_result&);
+ ThriftHiveMetastore_add_runtime_stats_result() {
+ }
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_result() throw();
+ MetaException o1;
+
+ _ThriftHiveMetastore_add_runtime_stats_result__isset __isset;
+
+ void __set_o1(const MetaException& val);
+
+ bool operator == (const ThriftHiveMetastore_add_runtime_stats_result & rhs)
const
+ {
+if (!(o1 == rhs.o1))
+ return false;
+return true;
+ }
+ bool operator != (const ThriftHiveMetastore_add_runtime_stats_result &rhs)
const {
+return !(*this == rhs);
+ }
+
+ bool operator < (const ThriftHiveMetastore_add_runtime_stats_result & )
const;
+
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+
+};
+
+typedef struct _ThriftHiveMetastore_add_runtime_stats_presult__isset {
+ _ThriftHiveMetastore_add_runtime_stats_presult__isset() : o1(false) {}
+ bool o1 :1;
+} _ThriftHiveMetastore_add_runtime_stats_presult__isset;
+
+class ThriftHiveMetastore_add_runtime_stats_presult {
+ public:
+
+
+ virtual ~ThriftHiveMetastore_add_runtime_stats_presult() throw();
+ MetaException o1;
+
+ _ThriftHiveMetastore_add_runtime_stats_presu
hive git commit: HIVE-19131: DecimalColumnStatsMergerTest comparison review (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master 5e0480e35 -> 334c8cae4
HIVE-19131: DecimalColumnStatsMergerTest comparison review (Laszlo Bodor via
Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/334c8cae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/334c8cae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/334c8cae
Branch: refs/heads/master
Commit: 334c8cae4af94be8157a3b9fd8506c8aee98ab50
Parents: 5e0480e
Author: Laszlo Bodor
Authored: Mon Apr 23 13:14:51 2018 +0200
Committer: Zoltan Haindrich
Committed: Mon Apr 23 13:14:51 2018 +0200
--
.../hive/ql/exec/ColumnStatsUpdateTask.java | 5 +-
.../ql/stats/ColumnStatisticsObjTranslator.java | 3 +-
.../stats_analyze_decimal_compare.q | 4 +
.../stats_analyze_decimal_compare.q.out | 45 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 40 ++---
.../gen/thrift/gen-cpp/hive_metastore_types.h | 12 +-
.../hadoop/hive/metastore/api/Decimal.java | 170 +--
.../src/gen/thrift/gen-php/metastore/Types.php | 34 ++--
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 24 +--
.../gen/thrift/gen-rb/hive_metastore_types.rb | 8 +-
.../hive/metastore/StatObjectConverter.java | 38 ++---
.../hive/metastore/api/utils/DecimalUtils.java | 49 ++
.../aggr/DecimalColumnStatsAggregator.java | 5 +-
.../src/main/thrift/hive_metastore.thrift | 4 +-
.../merge/DecimalColumnStatsMergerTest.java | 23 +--
15 files changed, 275 insertions(+), 189 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/334c8cae/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
index a7465a7..207b66f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
+import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
@@ -226,11 +227,11 @@ public class ColumnStatsUpdateTask extends
Task {
decimalStats.setNumDVs(Long.parseLong(value));
} else if (fName.equals("lowValue")) {
BigDecimal d = new BigDecimal(value);
- decimalStats.setLowValue(new Decimal(ByteBuffer.wrap(d
+ decimalStats.setLowValue(DecimalUtils.getDecimal(ByteBuffer.wrap(d
.unscaledValue().toByteArray()), (short) d.scale()));
} else if (fName.equals("highValue")) {
BigDecimal d = new BigDecimal(value);
- decimalStats.setHighValue(new Decimal(ByteBuffer.wrap(d
+ decimalStats.setHighValue(DecimalUtils.getDecimal(ByteBuffer.wrap(d
.unscaledValue().toByteArray()), (short) d.scale()));
} else {
throw new SemanticException("Unknown stat");
http://git-wip-us.apache.org/repos/asf/hive/blob/334c8cae/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
index 08cda4a..607545d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
@@ -28,6 +28,7 @@ import
org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
@@ -130,7 +131,7 @@ public class ColumnStatisticsObjTranslat
hive git commit: HIVE-19131: DecimalColumnStatsMergerTest comparison review (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/branch-3 4397f38c9 -> 818c8cd50
HIVE-19131: DecimalColumnStatsMergerTest comparison review (Laszlo Bodor via
Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/818c8cd5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/818c8cd5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/818c8cd5
Branch: refs/heads/branch-3
Commit: 818c8cd50c01e6b2ecac326608cd26226b30b6a4
Parents: 4397f38
Author: Laszlo Bodor
Authored: Mon Apr 23 13:14:51 2018 +0200
Committer: Zoltan Haindrich
Committed: Mon Apr 23 13:26:47 2018 +0200
--
.../hive/ql/exec/ColumnStatsUpdateTask.java | 5 +-
.../ql/stats/ColumnStatisticsObjTranslator.java | 3 +-
.../stats_analyze_decimal_compare.q | 4 +
.../stats_analyze_decimal_compare.q.out | 45 +
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 40 ++---
.../gen/thrift/gen-cpp/hive_metastore_types.h | 12 +-
.../hadoop/hive/metastore/api/Decimal.java | 170 +--
.../src/gen/thrift/gen-php/metastore/Types.php | 34 ++--
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 24 +--
.../gen/thrift/gen-rb/hive_metastore_types.rb | 8 +-
.../hive/metastore/StatObjectConverter.java | 38 ++---
.../hive/metastore/api/utils/DecimalUtils.java | 49 ++
.../aggr/DecimalColumnStatsAggregator.java | 5 +-
.../src/main/thrift/hive_metastore.thrift | 4 +-
.../merge/DecimalColumnStatsMergerTest.java | 23 +--
15 files changed, 275 insertions(+), 189 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/818c8cd5/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
index a7465a7..207b66f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsUpdateTask.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest;
+import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
@@ -226,11 +227,11 @@ public class ColumnStatsUpdateTask extends
Task {
decimalStats.setNumDVs(Long.parseLong(value));
} else if (fName.equals("lowValue")) {
BigDecimal d = new BigDecimal(value);
- decimalStats.setLowValue(new Decimal(ByteBuffer.wrap(d
+ decimalStats.setLowValue(DecimalUtils.getDecimal(ByteBuffer.wrap(d
.unscaledValue().toByteArray()), (short) d.scale()));
} else if (fName.equals("highValue")) {
BigDecimal d = new BigDecimal(value);
- decimalStats.setHighValue(new Decimal(ByteBuffer.wrap(d
+ decimalStats.setHighValue(DecimalUtils.getDecimal(ByteBuffer.wrap(d
.unscaledValue().toByteArray()), (short) d.scale()));
} else {
throw new SemanticException("Unknown stat");
http://git-wip-us.apache.org/repos/asf/hive/blob/818c8cd5/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
index 08cda4a..607545d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java
@@ -28,6 +28,7 @@ import
org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.api.Decimal;
+import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
import
org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
@@ -130,7 +131,7 @@ public class ColumnStatisticsObjTran
hive git commit: HIVE-19194: addendum for branch-3: fix imports
Repository: hive Updated Branches: refs/heads/branch-3 f29fae7d3 -> 4397f38c9 HIVE-19194: addendum for branch-3: fix imports Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4397f38c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4397f38c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4397f38c Branch: refs/heads/branch-3 Commit: 4397f38c9da60462a8cf1bd0dd6ed8dc7e745aba Parents: f29fae7 Author: Zoltan Haindrich Authored: Mon Apr 23 12:30:50 2018 +0200 Committer: Zoltan Haindrich Committed: Mon Apr 23 12:30:50 2018 +0200 -- .../apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java| 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/hive/blob/4397f38c/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java -- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 51b4cec..1f3eab7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import org.apache.hadoop.hive.ql.plan.PlanUtils; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.stats.StatsUtils; import java.io.FileNotFoundException;
hive git commit: HIVE-19009 : Retain and use runtime statistics during hs2 lifetime (Zoltan Haindrich via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 a39b24660 -> d0769c573
HIVE-19009 : Retain and use runtime statistics during hs2 lifetime (Zoltan
Haindrich via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan
(cherry picked from commit 9f15e22f4aea99891a37aa1e54d490921e6e1174)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d0769c57
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d0769c57
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d0769c57
Branch: refs/heads/branch-3
Commit: d0769c573b482fe440fbd4dd3c68d43f9c8f9524
Parents: a39b246
Author: Zoltan Haindrich
Authored: Tue Apr 3 08:51:00 2018 -0700
Committer: Zoltan Haindrich
Committed: Fri Apr 20 16:12:54 2018 +0200
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 11 +-
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/QTestUtil.java| 3 +
.../java/org/apache/hadoop/hive/ql/Context.java | 12 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 4 +
.../hive/ql/optimizer/physical/Vectorizer.java | 12 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java| 2 +-
.../hive/ql/plan/mapper/CachingStatsSource.java | 68 +
.../hive/ql/plan/mapper/EmptyStatsSource.java | 11 ++
.../plan/mapper/SimpleRuntimeStatsSource.java | 6 +
.../hadoop/hive/ql/plan/mapper/StatsSource.java | 5 +-
.../hive/ql/plan/mapper/StatsSources.java | 122
.../hive/ql/reexec/IReExecutionPlugin.java | 1 +
.../hadoop/hive/ql/reexec/ReExecDriver.java | 20 ++-
.../ql/reexec/ReExecutionOverlayPlugin.java | 4 +
.../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 48 +--
.../signature/TestOperatorSignature.java| 9 +-
.../ql/plan/mapping/TestCounterMapping.java | 1 -
.../queries/clientpositive/runtime_stats_hs2.q | 22 +++
.../clientpositive/llap/runtime_stats_hs2.q.out | 141 +++
20 files changed, 476 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/d0769c57/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7dd16e3..607f8ba 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4257,10 +4257,19 @@ public class HiveConf extends Configuration {
"comma separated list of plugin can be used:\n"
+ " overlay: hiveconf subtree 'reexec.overlay' is used as an
overlay in case of an execution errors out\n"
+ " reoptimize: collects operator statistics during execution and
recompile the query after a failure"),
+
HIVE_QUERY_REEXECUTION_STATS_PERSISTENCE("hive.query.reexecution.stats.persist.scope",
"query",
+new StringSet("query", "hiveserver", "metastore"),
+"Sets the persistence scope of runtime statistics\n"
++ " query: runtime statistics are only used during re-execution\n"
++ " hiveserver: runtime statistics are persisted in the
hiveserver - all sessions share it"),
+
HIVE_QUERY_MAX_REEXECUTION_COUNT("hive.query.reexecution.max.count", 1,
"Maximum number of re-executions for a single query."),
HIVE_QUERY_REEXECUTION_ALWAYS_COLLECT_OPERATOR_STATS("hive.query.reexecution.always.collect.operator.stats",
false,
-"Used during testing"),
+"If sessionstats are enabled; this option can be used to collect
statistics all the time"),
+
HIVE_QUERY_REEXECUTION_STATS_CACHE_SIZE("hive.query.reexecution.stats.cache.size",
100_000,
+"Size of the runtime statistics cache. Unit is: OperatorStat entry; a
query plan consist ~100"),
+
HIVE_QUERY_RESULTS_CACHE_ENABLED("hive.query.results.cache.enabled", true,
"If the query results cache is enabled. This will keep results of
previously executed queries " +
http://git-wip-us.apache.org/repos/asf/hive/blob/d0769c57/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 183dc4c..d7c5877 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -514,6 +514,7 @@ minillaplocal.query.files=\
retry_failure.q,\
retry_failure_stat_changes.q,\
retry_failure_oom.q,\
+ runtime_stats_hs2.q,\
bucketsortoptimize_insert_2.q,\
check_constraint.q,\
cbo_gby.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/d0769c57/itests/util
hive git commit: HIVE-18946: Fix columnstats merge NPE (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/branch-3 013403392 -> 0946ab207
HIVE-18946: Fix columnstats merge NPE (Laszlo Bodor via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0946ab20
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0946ab20
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0946ab20
Branch: refs/heads/branch-3
Commit: 0946ab207f8031872f165c0cbd1e3c3c1e0f4cda
Parents: 0134033
Author: Laszlo Bodor
Authored: Tue Apr 17 11:05:39 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue Apr 17 11:07:34 2018 +0200
--
.../clientpositive/stats_analyze_empty.q| 18 ++
.../clientpositive/stats_analyze_empty.q.out| 48
.../merge/DecimalColumnStatsMerger.java | 36 ++-
.../merge/DecimalColumnStatsMergerTest.java | 242 +++
4 files changed, 338 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/0946ab20/ql/src/test/queries/clientpositive/stats_analyze_empty.q
--
diff --git a/ql/src/test/queries/clientpositive/stats_analyze_empty.q
b/ql/src/test/queries/clientpositive/stats_analyze_empty.q
new file mode 100644
index 000..6ea6125
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stats_analyze_empty.q
@@ -0,0 +1,18 @@
+set hive.stats.autogather=true;
+set hive.explain.user=true;
+
+drop table if exists testdeci2;
+
+create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+stored as orc location '/tmp/testdeci2'
+TBLPROPERTIES ("transactional"="false")
+;
+
+
+analyze table testdeci2 compute statistics for columns;
+
+insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2');
http://git-wip-us.apache.org/repos/asf/hive/blob/0946ab20/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
--
diff --git a/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
b/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
new file mode 100644
index 000..6eb51e9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
@@ -0,0 +1,48 @@
+PREHOOK: query: drop table if exists testdeci2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testdeci2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+ A masked pattern was here
+TBLPROPERTIES ("transactional"="false")
+PREHOOK: type: CREATETABLE
+ A masked pattern was here
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testdeci2
+POSTHOOK: query: create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+ A masked pattern was here
+TBLPROPERTIES ("transactional"="false")
+POSTHOOK: type: CREATETABLE
+ A masked pattern was here
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testdeci2
+PREHOOK: query: analyze table testdeci2 compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testdeci2
+PREHOOK: Output: default@testdeci2
+ A masked pattern was here
+POSTHOOK: query: analyze table testdeci2 compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testdeci2
+POSTHOOK: Output: default@testdeci2
+ A masked pattern was here
+PREHOOK: query: insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@testdeci2
+POSTHOOK: query: insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@testdeci2
+POSTHOOK: Lineage: testdeci2.amount SCRIPT []
+POSTHOOK: Lineage: testdeci2.id SCRIPT []
+POSTHOOK: Lineage: testdeci2.item SCRIPT []
+POSTHOOK: Lineage: testdeci2.sales_tax SCRIPT []
http://git-wip-us.apache.org/repos/asf/hive/blob/0946ab20/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
index 01f3385..517ca72 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
+++
b/standalone-metastore/s
hive git commit: HIVE-18946: Fix columnstats merge NPE (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master fa9e743e7 -> 54046353e
HIVE-18946: Fix columnstats merge NPE (Laszlo Bodor via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/54046353
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/54046353
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/54046353
Branch: refs/heads/master
Commit: 54046353e022d7aa5f3656de7d2a66f15608466d
Parents: fa9e743
Author: Laszlo Bodor
Authored: Tue Apr 17 11:05:39 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue Apr 17 11:06:33 2018 +0200
--
.../clientpositive/stats_analyze_empty.q| 18 ++
.../clientpositive/stats_analyze_empty.q.out| 48
.../merge/DecimalColumnStatsMerger.java | 36 ++-
.../merge/DecimalColumnStatsMergerTest.java | 242 +++
4 files changed, 338 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/54046353/ql/src/test/queries/clientpositive/stats_analyze_empty.q
--
diff --git a/ql/src/test/queries/clientpositive/stats_analyze_empty.q
b/ql/src/test/queries/clientpositive/stats_analyze_empty.q
new file mode 100644
index 000..6ea6125
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stats_analyze_empty.q
@@ -0,0 +1,18 @@
+set hive.stats.autogather=true;
+set hive.explain.user=true;
+
+drop table if exists testdeci2;
+
+create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+stored as orc location '/tmp/testdeci2'
+TBLPROPERTIES ("transactional"="false")
+;
+
+
+analyze table testdeci2 compute statistics for columns;
+
+insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2');
http://git-wip-us.apache.org/repos/asf/hive/blob/54046353/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
--
diff --git a/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
b/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
new file mode 100644
index 000..6eb51e9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/stats_analyze_empty.q.out
@@ -0,0 +1,48 @@
+PREHOOK: query: drop table if exists testdeci2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testdeci2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+ A masked pattern was here
+TBLPROPERTIES ("transactional"="false")
+PREHOOK: type: CREATETABLE
+ A masked pattern was here
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testdeci2
+POSTHOOK: query: create table testdeci2(
+id int,
+amount decimal(10,3),
+sales_tax decimal(10,3),
+item string)
+ A masked pattern was here
+TBLPROPERTIES ("transactional"="false")
+POSTHOOK: type: CREATETABLE
+ A masked pattern was here
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testdeci2
+PREHOOK: query: analyze table testdeci2 compute statistics for columns
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testdeci2
+PREHOOK: Output: default@testdeci2
+ A masked pattern was here
+POSTHOOK: query: analyze table testdeci2 compute statistics for columns
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testdeci2
+POSTHOOK: Output: default@testdeci2
+ A masked pattern was here
+PREHOOK: query: insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@testdeci2
+POSTHOOK: query: insert into table testdeci2
values(1,12.123,12345.123,'desk1'),(2,123.123,1234.123,'desk2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@testdeci2
+POSTHOOK: Lineage: testdeci2.amount SCRIPT []
+POSTHOOK: Lineage: testdeci2.id SCRIPT []
+POSTHOOK: Lineage: testdeci2.item SCRIPT []
+POSTHOOK: Lineage: testdeci2.sales_tax SCRIPT []
http://git-wip-us.apache.org/repos/asf/hive/blob/54046353/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
index 01f3385..517ca72 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
+++
b/standalone-metastore/src/m
[04/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_null.q -- diff --git a/ql/src/test/queries/clientpositive/union_null.q b/ql/src/test/queries/clientpositive/union_null.q index aacf310..49a0e08 100644 --- a/ql/src/test/queries/clientpositive/union_null.q +++ b/ql/src/test/queries/clientpositive/union_null.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src -- SORT_BEFORE_DIFF -- HIVE-2901 http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_paren.q -- diff --git a/ql/src/test/queries/clientpositive/union_paren.q b/ql/src/test/queries/clientpositive/union_paren.q index 0b38b68..f655a87 100644 --- a/ql/src/test/queries/clientpositive/union_paren.q +++ b/ql/src/test/queries/clientpositive/union_paren.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; explain select * from src union all select * from src; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_pos_alias.q -- diff --git a/ql/src/test/queries/clientpositive/union_pos_alias.q b/ql/src/test/queries/clientpositive/union_pos_alias.q index 60fea8f..e924d5d 100644 --- a/ql/src/test/queries/clientpositive/union_pos_alias.q +++ b/ql/src/test/queries/clientpositive/union_pos_alias.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_ppr.q -- diff --git a/ql/src/test/queries/clientpositive/union_ppr.q b/ql/src/test/queries/clientpositive/union_ppr.q index 013e147..09f6b97 100644 --- a/ql/src/test/queries/clientpositive/union_ppr.q +++ b/ql/src/test/queries/clientpositive/union_ppr.q @@ -1,3 +1,4 @@ +--! qt:dataset:srcpart -- SORT_QUERY_RESULTS EXPLAIN EXTENDED http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_remove_25.q -- diff --git a/ql/src/test/queries/clientpositive/union_remove_25.q b/ql/src/test/queries/clientpositive/union_remove_25.q index 78fa9aa..0460eb3 100644 --- a/ql/src/test/queries/clientpositive/union_remove_25.q +++ b/ql/src/test/queries/clientpositive/union_remove_25.q @@ -1,3 +1,4 @@ +--! qt:dataset:srcpart set hive.mapred.mode=nonstrict; set hive.stats.autogather=false; set hive.optimize.union.remove=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_remove_6_subq.q -- diff --git a/ql/src/test/queries/clientpositive/union_remove_6_subq.q b/ql/src/test/queries/clientpositive/union_remove_6_subq.q index 3607a52..2dd4716 100644 --- a/ql/src/test/queries/clientpositive/union_remove_6_subq.q +++ b/ql/src/test/queries/clientpositive/union_remove_6_subq.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.stats.autogather=false; set hive.optimize.union.remove=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_script.q -- diff --git a/ql/src/test/queries/clientpositive/union_script.q b/ql/src/test/queries/clientpositive/union_script.q index e5b160f..7b563d3 100644 --- a/ql/src/test/queries/clientpositive/union_script.q +++ b/ql/src/test/queries/clientpositive/union_script.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; -- SORT_QUERY_RESULTS select * from ( http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_stats.q -- diff --git a/ql/src/test/queries/clientpositive/union_stats.q b/ql/src/test/queries/clientpositive/union_stats.q index 789b360..0e91c23 100644 --- a/ql/src/test/queries/clientpositive/union_stats.q +++ b/ql/src/test/queries/clientpositive/union_stats.q @@ -1,3 +1,4 @@ +--! qt:dataset:src explain extended create table t as select * from src union all select * from src; create table t as select * from src union all select * from src; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/union_top_level.q -- diff --git a/ql/src/test/queries/clientpositive/union_top_level.q b/ql/src/test/queries/clientpositive/union_top_level.q index 2050442..cdb1628 100644 --- a/ql/src/test/queries/clientpositive/union_top_level.q +++ b/ql/src/test/queries/clientpositive/union_top_level.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.
[14/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/ansi_sql_arithmetic.q -- diff --git a/ql/src/test/queries/clientpositive/ansi_sql_arithmetic.q b/ql/src/test/queries/clientpositive/ansi_sql_arithmetic.q index b343d78..dc235df 100644 --- a/ql/src/test/queries/clientpositive/ansi_sql_arithmetic.q +++ b/ql/src/test/queries/clientpositive/ansi_sql_arithmetic.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.compat=latest; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q -- diff --git a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q index e961641..75bd579 100644 --- a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q +++ b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q @@ -1,3 +1,5 @@ +--! qt:dataset:srcpart +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.archive.enabled = true; set hive.exec.submitviachild=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/archive_multi.q -- diff --git a/ql/src/test/queries/clientpositive/archive_multi.q b/ql/src/test/queries/clientpositive/archive_multi.q index c18f991..60cb4a9 100644 --- a/ql/src/test/queries/clientpositive/archive_multi.q +++ b/ql/src/test/queries/clientpositive/archive_multi.q @@ -1,3 +1,5 @@ +--! qt:dataset:srcpart +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.archive.enabled = true; ; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q -- diff --git a/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q b/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q index ea6e9d5..154641e 100644 --- a/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q +++ b/ql/src/test/queries/clientpositive/array_map_access_nonconstant.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.fetch.task.conversion=more; create table array_table (`array` array, index int ); http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/authorization_1.q -- diff --git a/ql/src/test/queries/clientpositive/authorization_1.q b/ql/src/test/queries/clientpositive/authorization_1.q index 184acc7..42abff0 100644 --- a/ql/src/test/queries/clientpositive/authorization_1.q +++ b/ql/src/test/queries/clientpositive/authorization_1.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; -- SORT_BEFORE_DIFF http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/authorization_2.q -- diff --git a/ql/src/test/queries/clientpositive/authorization_2.q b/ql/src/test/queries/clientpositive/authorization_2.q index efb42f7..942b09f 100644 --- a/ql/src/test/queries/clientpositive/authorization_2.q +++ b/ql/src/test/queries/clientpositive/authorization_2.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; -- SORT_BEFORE_DIFF http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/authorization_3.q -- diff --git a/ql/src/test/queries/clientpositive/authorization_3.q b/ql/src/test/queries/clientpositive/authorization_3.q index 09c6494..95c25fe 100644 --- a/ql/src/test/queries/clientpositive/authorization_3.q +++ b/ql/src/test/queries/clientpositive/authorization_3.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; -- SORT_BEFORE_DIFF http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/authorization_4.q -- diff --git a/ql/src/test/queries/clientpositive/authorization_4.q b/ql/src/test/queries/clientpositive/authorization_4.q index c1848a7..f76ba1f 100644 --- a/ql/src/test/queries/clientpositive/authorization_4.q +++ b/ql/src/test/queries/clientpositive/authorization_4.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; -- SORT_BEFORE_DIFF htt
[05/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_character_length.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_character_length.q
b/ql/src/test/queries/clientpositive/udf_character_length.q
index 1312334..fe0c1ba 100644
--- a/ql/src/test/queries/clientpositive/udf_character_length.q
+++ b/ql/src/test/queries/clientpositive/udf_character_length.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src1
SET hive.vectorized.execution.enabled=false;
set hive.fetch.task.conversion=more;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_classloader.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_classloader.q
b/ql/src/test/queries/clientpositive/udf_classloader.q
index 82678c2..00ad858 100644
--- a/ql/src/test/queries/clientpositive/udf_classloader.q
+++ b/ql/src/test/queries/clientpositive/udf_classloader.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
ADD JAR
${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf1/${system:hive.version}/udf-classloader-udf1-${system:hive.version}.jar;
ADD JAR
${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-util/${system:hive.version}/udf-classloader-util-${system:hive.version}.jar;
ADD JAR
${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
--
diff --git
a/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
index 2ceeaa3..19772fc 100644
---
a/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
+++
b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf1:+;
ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-util:+;
ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_coalesce.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_coalesce.q
b/ql/src/test/queries/clientpositive/udf_coalesce.q
index d3c417b..7248efe 100644
--- a/ql/src/test/queries/clientpositive/udf_coalesce.q
+++ b/ql/src/test/queries/clientpositive/udf_coalesce.q
@@ -1,3 +1,5 @@
+--! qt:dataset:src_thrift
+--! qt:dataset:src
set hive.fetch.task.conversion=more;
DESCRIBE FUNCTION coalesce;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_compare_java_string.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_compare_java_string.q
b/ql/src/test/queries/clientpositive/udf_compare_java_string.q
index c7983b8..320296c 100644
--- a/ql/src/test/queries/clientpositive/udf_compare_java_string.q
+++ b/ql/src/test/queries/clientpositive/udf_compare_java_string.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.fetch.task.conversion=more;
EXPLAIN
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_concat.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_concat.q
b/ql/src/test/queries/clientpositive/udf_concat.q
index e35a1cf..ea0a907 100644
--- a/ql/src/test/queries/clientpositive/udf_concat.q
+++ b/ql/src/test/queries/clientpositive/udf_concat.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.fetch.task.conversion=more;
DESCRIBE FUNCTION concat;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_concat_insert1.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_concat_insert1.q
b/ql/src/test/queries/clientpositive/udf_concat_insert1.q
index 496f403..dd639cb 100644
--- a/ql/src/test/queries/clientpositive/udf_concat_insert1.q
+++ b/ql/src/test/queries/clientpositive/udf_concat_insert1.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/udf_concat_insert2.q
--
diff --git a/ql/src/test/queries/clientpositive/udf_concat_insert2.q
b/ql/src/tes
[11/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q b/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q index 0a1f137..26236ab 100644 --- a/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q +++ b/ql/src/test/queries/clientpositive/groupby_complex_types_multi_single_reducer.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.multigroupby.singlereducer=true; -- SORT_QUERY_RESULTS http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_constcolval.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_constcolval.q b/ql/src/test/queries/clientpositive/groupby_constcolval.q index e0ad694..a844258 100644 --- a/ql/src/test/queries/clientpositive/groupby_constcolval.q +++ b/ql/src/test/queries/clientpositive/groupby_constcolval.q @@ -1,3 +1,4 @@ +--! qt:dataset:src DROP TABLE IF EXISTS tlbtest; create table tlbtest (key int, key1 int, key2 int); select key, key1, key2 from (select a.key, 0 as key1 , 0 as key2 from tlbtest a inner join src b on a.key = b.key) a group by key, key1, key2; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q b/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q index dff81a4..b2a0dfa 100644 --- a/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q +++ b/ql/src/test/queries/clientpositive/groupby_cube_multi_gby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.multigroupby.singlereducer=false; create table t1 like src; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_distinct_samekey.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_distinct_samekey.q b/ql/src/test/queries/clientpositive/groupby_distinct_samekey.q index 6a44dd1..7a40d01 100644 --- a/ql/src/test/queries/clientpositive/groupby_distinct_samekey.q +++ b/ql/src/test/queries/clientpositive/groupby_distinct_samekey.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; -- This test covers HIVE-2332 -- SORT_QUERY_RESULTS http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_duplicate_key.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_duplicate_key.q b/ql/src/test/queries/clientpositive/groupby_duplicate_key.q index 1909873..72e38f4 100644 --- a/ql/src/test/queries/clientpositive/groupby_duplicate_key.q +++ b/ql/src/test/queries/clientpositive/groupby_duplicate_key.q @@ -1,3 +1,4 @@ +--! qt:dataset:src explain select distinct key, "" as dummy1, "" as dummy2 from src tablesample (10 rows); http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_grouping_window.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_grouping_window.q b/ql/src/test/queries/clientpositive/groupby_grouping_window.q index 7ba73f9..8a5c290 100644 --- a/ql/src/test/queries/clientpositive/groupby_grouping_window.q +++ b/ql/src/test/queries/clientpositive/groupby_grouping_window.q @@ -1,3 +1,4 @@ +--! qt:dataset:src create table t(category int, live int, comments int); insert into table t select key, 0, 2 from src tablesample(3 rows); http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_join_pushdown.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_join_pushdown.q b/ql/src/test/queries/clientpositive/groupby_join_pushdown.q index d0bf0fb..37ecd90 100644 --- a/ql/src/test/queries/clientpositive/groupby_join_pushdown.q +++ b/ql/src/test/queries/clientpositive/groupby_join_pushdown.q @@ -1,3 +1,5 @@ +--! qt:dataset:src +--! qt:dataset:alltypesorc set hive.mapred.mode=nonstrict; set hive.transpose.aggr.join=true; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/groupby_map_ppr.q -- diff --git a/ql/src/test/queries/clientpositive/groupby_map_ppr.q b/ql/src/test/queries/clientpositive/groupby_map_ppr.q index cec370f..d42416e 100644 --- a/ql/src/test/queries/clientpositive/groupby_map_ppr.q +++ b/ql/src/test/queries/clientpositive/groupby_map_ppr.q @@ -1
[08/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_schema_evolution.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q index e767b81..00cad9e 100644 --- a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q +++ b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q @@ -1,3 +1,4 @@ +--! qt:dataset:srcpart set hive.mapred.mode=nonstrict; -- Some tables might have extra columns and struct elements on the schema than the on Parquet schema; -- This is called 'schema evolution' as the Parquet file is not ready yet for such new columns; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_serde.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_serde.q b/ql/src/test/queries/clientpositive/parquet_serde.q index 275f92a..9661f5b 100644 --- a/ql/src/test/queries/clientpositive/parquet_serde.q +++ b/ql/src/test/queries/clientpositive/parquet_serde.q @@ -1,3 +1,4 @@ +--! qt:dataset:part set hive.mapred.mode=nonstrict; DROP TABLE if exists parquet_mixed_fileformat; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_0.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_0.q b/ql/src/test/queries/clientpositive/parquet_vectorization_0.q index 69628b8..fa8ec86 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_0.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_0.q @@ -1,3 +1,6 @@ +--! qt:dataset:srcbucket +--! qt:dataset:src +--! qt:dataset:alltypesparquet set hive.mapred.mode=nonstrict; set hive.explain.user=false; SET hive.vectorized.execution.enabled=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_1.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_1.q b/ql/src/test/queries/clientpositive/parquet_vectorization_1.q index 95b2c9e..311334d 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_1.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_1.q @@ -1,3 +1,4 @@ +--! qt:dataset:alltypesparquet SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; set hive.vectorized.execution.reduce.enabled=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_10.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_10.q b/ql/src/test/queries/clientpositive/parquet_vectorization_10.q index 14077e3..c560515 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_10.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_10.q @@ -1,3 +1,4 @@ +--! qt:dataset:alltypesparquet SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_11.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_11.q b/ql/src/test/queries/clientpositive/parquet_vectorization_11.q index 48ddecf..a6e6810 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_11.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_11.q @@ -1,3 +1,4 @@ +--! qt:dataset:alltypesparquet SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_12.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_12.q b/ql/src/test/queries/clientpositive/parquet_vectorization_12.q index e3606e6..ff6949a 100644 --- a/ql/src/test/queries/clientpositive/parquet_vectorization_12.q +++ b/ql/src/test/queries/clientpositive/parquet_vectorization_12.q @@ -1,3 +1,4 @@ +--! qt:dataset:alltypesparquet set hive.mapred.mode=nonstrict; SET hive.vectorized.execution.enabled=true; set hive.fetch.task.conversion=none; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/parquet_vectorization_13.q -- diff --git a/ql/src/test/queries/clientpositive/parquet_vectorization_13.q b/ql/src/test/queries/clientpositive/parquet_vectorization_13.q index 0763330..e5f48c8 100644 --- a/q
[07/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/reduce_deduplicate_exclude_join.q
--
diff --git
a/ql/src/test/queries/clientpositive/reduce_deduplicate_exclude_join.q
b/ql/src/test/queries/clientpositive/reduce_deduplicate_exclude_join.q
index ff09754..93e1cb4 100644
--- a/ql/src/test/queries/clientpositive/reduce_deduplicate_exclude_join.q
+++ b/ql/src/test/queries/clientpositive/reduce_deduplicate_exclude_join.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.optimize.reducededuplication=true;
set hive.auto.convert.join=true;
explain select * from (select * from src cluster by key) a join src b on a.key
= b.key limit 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
--
diff --git a/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
b/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
index 8c9ff66..f05fa9f 100644
--- a/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
+++ b/ql/src/test/queries/clientpositive/reduce_deduplicate_extended.q
@@ -1,3 +1,5 @@
+--! qt:dataset:src1
+--! qt:dataset:src
set hive.mapred.mode=nonstrict;
set hive.optimize.reducededuplication=true;
set hive.optimize.reducededuplication.min.reducer=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q
--
diff --git a/ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q
b/ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q
index cd67f4c..f88fd50 100644
--- a/ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q
+++ b/ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.mapred.mode=nonstrict;
set hive.explain.user=false;
set hive.auto.convert.join=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/reducesink_dedup.q
--
diff --git a/ql/src/test/queries/clientpositive/reducesink_dedup.q
b/ql/src/test/queries/clientpositive/reducesink_dedup.q
index d429a1f..352a558 100644
--- a/ql/src/test/queries/clientpositive/reducesink_dedup.q
+++ b/ql/src/test/queries/clientpositive/reducesink_dedup.q
@@ -1,3 +1,4 @@
+--! qt:dataset:part
select p_name
from (select p_name from part distribute by 1 sort by 1) p
distribute by 1 sort by 1
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/regex_col.q
--
diff --git a/ql/src/test/queries/clientpositive/regex_col.q
b/ql/src/test/queries/clientpositive/regex_col.q
index 774cb62..d422f75 100644
--- a/ql/src/test/queries/clientpositive/regex_col.q
+++ b/ql/src/test/queries/clientpositive/regex_col.q
@@ -1,3 +1,4 @@
+--! qt:dataset:srcpart
set hive.mapred.mode=nonstrict;
set hive.support.quoted.identifiers=none;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/regexp_extract.q
--
diff --git a/ql/src/test/queries/clientpositive/regexp_extract.q
b/ql/src/test/queries/clientpositive/regexp_extract.q
index 2074eea..678b8fa 100644
--- a/ql/src/test/queries/clientpositive/regexp_extract.q
+++ b/ql/src/test/queries/clientpositive/regexp_extract.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
EXPLAIN EXTENDED
FROM (
FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/reloadJar.q
--
diff --git a/ql/src/test/queries/clientpositive/reloadJar.q
b/ql/src/test/queries/clientpositive/reloadJar.q
index 6768a4f..e7a50e1 100644
--- a/ql/src/test/queries/clientpositive/reloadJar.q
+++ b/ql/src/test/queries/clientpositive/reloadJar.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
dfs -mkdir ${system:test.tmp.dir}/aux;
dfs -cp ${system:hive.root}/data/files/identity_udf.jar
${system:test.tmp.dir}/aux/udfexample.jar;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/remote_script.q
--
diff --git a/ql/src/test/queries/clientpositive/remote_script.q
b/ql/src/test/queries/clientpositive/remote_script.q
index ba0d00f..19657bc 100644
--- a/ql/src/test/queries/clientpositive/remote_script.q
+++ b/ql/src/test/queries/clientpositive/remote_script.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.mapred.mode=nonstrict;
dfs -put ../../data/scripts/newline.py /newline.py;
add file hdfs:///newline.py;
http://git-wip-us.apache.org/repos/asf
[12/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/decimal_stats.q
--
diff --git a/ql/src/test/queries/clientpositive/decimal_stats.q
b/ql/src/test/queries/clientpositive/decimal_stats.q
index e00cf1f..ab87ad6 100644
--- a/ql/src/test/queries/clientpositive/decimal_stats.q
+++ b/ql/src/test/queries/clientpositive/decimal_stats.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.stats.fetch.column.stats=true;
drop table if exists decimal_1;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
b/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
index 0638537..91bdd4b 100644
--- a/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
+++ b/ql/src/test/queries/clientpositive/delete_all_non_partitioned.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_all_partitioned.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_all_partitioned.q
b/ql/src/test/queries/clientpositive/delete_all_partitioned.q
index 3c41ed5..001e30e 100644
--- a/ql/src/test/queries/clientpositive/delete_all_partitioned.q
+++ b/ql/src/test/queries/clientpositive/delete_all_partitioned.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_orig_table.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_orig_table.q
b/ql/src/test/queries/clientpositive/delete_orig_table.q
index c73b21c..c009b76 100644
--- a/ql/src/test/queries/clientpositive/delete_orig_table.q
+++ b/ql/src/test/queries/clientpositive/delete_orig_table.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
-- Suppress vectorization due to known bug. See HIVE-19109.
set hive.vectorized.execution.enabled=false;
set hive.test.vectorized.execution.enabled.override=disable;
@@ -7,7 +8,7 @@ set
hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/delete_orig_table;
-dfs -copyFromLocal ../../data/files/alltypesorc
${system:test.tmp.dir}/delete_orig_table/0_0;
+dfs -copyFromLocal ../../data/files/alltypesorc
${system:test.tmp.dir}/delete_orig_table/0_0;
create table acid_dot(
ctinyint TINYINT,
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_tmp_table.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_tmp_table.q
b/ql/src/test/queries/clientpositive/delete_tmp_table.q
index 49e586f..1aea4aa 100644
--- a/ql/src/test/queries/clientpositive/delete_tmp_table.q
+++ b/ql/src/test/queries/clientpositive/delete_tmp_table.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_where_no_match.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_where_no_match.q
b/ql/src/test/queries/clientpositive/delete_where_no_match.q
index e558619..844c586 100644
--- a/ql/src/test/queries/clientpositive/delete_where_no_match.q
+++ b/ql/src/test/queries/clientpositive/delete_where_no_match.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/delete_where_non_partitioned.q
--
diff --git a/ql/src/test/queries/clientpositive/delete_where_non_partitioned.q
b/ql/src/test/queries/clientpositive/delete_where_non_partitioned.q
index 73137bf..cacde21 100644
--- a/ql/src/test/queries/clientpositive/delete_where_non_partitioned.q
+++ b/ql/src/test/queries/clientpositive/delete_where_non_partitioned.q
@@ -1,3 +1,4 @@
+--! qt:dataset:alltypesorc
set hive.mapred.mode=nonstrict;
set hive.support.concurr
[13/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_gby.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_gby.q b/ql/src/test/queries/clientpositive/cbo_gby.q index 7aca53a..cab45db 100644 --- a/ql/src/test/queries/clientpositive/cbo_gby.q +++ b/ql/src/test/queries/clientpositive/cbo_gby.q @@ -1,3 +1,6 @@ +--! qt:dataset:cbo_t3 +--! qt:dataset:cbo_t2 +--! qt:dataset:cbo_t1 set hive.mapred.mode=nonstrict; set hive.cbo.enable=true; set hive.exec.check.crossproducts=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_gby_empty.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_gby_empty.q b/ql/src/test/queries/clientpositive/cbo_gby_empty.q index 7b23d01..3232932 100644 --- a/ql/src/test/queries/clientpositive/cbo_gby_empty.q +++ b/ql/src/test/queries/clientpositive/cbo_gby_empty.q @@ -1,3 +1,5 @@ +--! qt:dataset:src +--! qt:dataset:cbo_t3 set hive.mapred.mode=nonstrict; set hive.cbo.enable=true; set hive.exec.check.crossproducts=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_input26.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_input26.q b/ql/src/test/queries/clientpositive/cbo_input26.q index 40050f9..7ddce75 100644 --- a/ql/src/test/queries/clientpositive/cbo_input26.q +++ b/ql/src/test/queries/clientpositive/cbo_input26.q @@ -1,3 +1,4 @@ +--! qt:dataset:srcpart set hive.mapred.mode=nonstrict; set hive.optimize.constant.propagation=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_join.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_join.q b/ql/src/test/queries/clientpositive/cbo_join.q index eef2440..fb3fce7 100644 --- a/ql/src/test/queries/clientpositive/cbo_join.q +++ b/ql/src/test/queries/clientpositive/cbo_join.q @@ -1,3 +1,6 @@ +--! qt:dataset:cbo_t3 +--! qt:dataset:cbo_t2 +--! qt:dataset:cbo_t1 set hive.mapred.mode=nonstrict; set hive.cbo.enable=true; set hive.exec.check.crossproducts=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_limit.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_limit.q b/ql/src/test/queries/clientpositive/cbo_limit.q index 5e54a1b..24c1eae 100644 --- a/ql/src/test/queries/clientpositive/cbo_limit.q +++ b/ql/src/test/queries/clientpositive/cbo_limit.q @@ -1,3 +1,6 @@ +--! qt:dataset:cbo_t3 +--! qt:dataset:cbo_t2 +--! qt:dataset:cbo_t1 set hive.mapred.mode=nonstrict; set hive.cbo.enable=true; set hive.exec.check.crossproducts=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_rp_auto_join0.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_rp_auto_join0.q b/ql/src/test/queries/clientpositive/cbo_rp_auto_join0.q index e64fd41..5929ad7 100644 --- a/ql/src/test/queries/clientpositive/cbo_rp_auto_join0.q +++ b/ql/src/test/queries/clientpositive/cbo_rp_auto_join0.q @@ -1,3 +1,6 @@ +--! qt:dataset:cbo_t3 +--! qt:dataset:cbo_t2 +--! qt:dataset:cbo_t1 set hive.mapred.mode=nonstrict; set hive.cbo.returnpath.hiveop=true; set hive.stats.fetch.column.stats=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_rp_auto_join1.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_rp_auto_join1.q b/ql/src/test/queries/clientpositive/cbo_rp_auto_join1.q index e9e434c..9137669 100644 --- a/ql/src/test/queries/clientpositive/cbo_rp_auto_join1.q +++ b/ql/src/test/queries/clientpositive/cbo_rp_auto_join1.q @@ -1,3 +1,6 @@ +--! qt:dataset:src1 +--! qt:dataset:src +--! qt:dataset:part set hive.cbo.returnpath.hiveop=true; set hive.stats.fetch.column.stats=true; set hive.optimize.semijoin.conversion=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/cbo_rp_auto_join17.q -- diff --git a/ql/src/test/queries/clientpositive/cbo_rp_auto_join17.q b/ql/src/test/queries/clientpositive/cbo_rp_auto_join17.q index f7da842..ae17ec6 100644 --- a/ql/src/test/queries/clientpositive/cbo_rp_auto_join17.q +++ b/ql/src/test/queries/clientpositive/cbo_rp_auto_join17.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.cbo.returnpath.hiveop=true; set hive.auto.convert.join = true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/tes
[10/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join16.q -- diff --git a/ql/src/test/queries/clientpositive/join16.q b/ql/src/test/queries/clientpositive/join16.q index 901f9ca..91565c7 100644 --- a/ql/src/test/queries/clientpositive/join16.q +++ b/ql/src/test/queries/clientpositive/join16.q @@ -1,2 +1,3 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; EXPLAIN SELECT subq.key, tab.value FROM (select a.key, a.value from src a where a.key > 10 ) subq JOIN src tab ON (subq.key = tab.key and subq.key > 20 and subq.value = tab.value) where tab.value < 200; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join17.q -- diff --git a/ql/src/test/queries/clientpositive/join17.q b/ql/src/test/queries/clientpositive/join17.q index 4a7eb99..a62b004 100644 --- a/ql/src/test/queries/clientpositive/join17.q +++ b/ql/src/test/queries/clientpositive/join17.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src set hive.mapred.mode=nonstrict; -- SORT_QUERY_RESULTS http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join18.q -- diff --git a/ql/src/test/queries/clientpositive/join18.q b/ql/src/test/queries/clientpositive/join18.q index 702735d..9ac2f2f 100644 --- a/ql/src/test/queries/clientpositive/join18.q +++ b/ql/src/test/queries/clientpositive/join18.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src set hive.mapred.mode=nonstrict; -- SORT_QUERY_RESULTS http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join18_multi_distinct.q -- diff --git a/ql/src/test/queries/clientpositive/join18_multi_distinct.q b/ql/src/test/queries/clientpositive/join18_multi_distinct.q index 8b3ac4d..cbdf27e 100644 --- a/ql/src/test/queries/clientpositive/join18_multi_distinct.q +++ b/ql/src/test/queries/clientpositive/join18_multi_distinct.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src -- SORT_QUERY_RESULTS EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join2.q -- diff --git a/ql/src/test/queries/clientpositive/join2.q b/ql/src/test/queries/clientpositive/join2.q index c3c7c24..074255bb 100644 --- a/ql/src/test/queries/clientpositive/join2.q +++ b/ql/src/test/queries/clientpositive/join2.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src -- due to testMTQueries1 set hive.stats.column.autogather=false; set hive.mapred.mode=nonstrict; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join20.q -- diff --git a/ql/src/test/queries/clientpositive/join20.q b/ql/src/test/queries/clientpositive/join20.q index b2b1eba..199c4b4 100644 --- a/ql/src/test/queries/clientpositive/join20.q +++ b/ql/src/test/queries/clientpositive/join20.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src -- SORT_QUERY_RESULTS EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join21.q -- diff --git a/ql/src/test/queries/clientpositive/join21.q b/ql/src/test/queries/clientpositive/join21.q index 8ba2284..796fe80 100644 --- a/ql/src/test/queries/clientpositive/join21.q +++ b/ql/src/test/queries/clientpositive/join21.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src -- SORT_QUERY_RESULTS EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join22.q -- diff --git a/ql/src/test/queries/clientpositive/join22.q b/ql/src/test/queries/clientpositive/join22.q index 525adfe..3a10cfc 100644 --- a/ql/src/test/queries/clientpositive/join22.q +++ b/ql/src/test/queries/clientpositive/join22.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src set hive.mapred.mode=nonstrict; explain SELECT src5.src1_value FROM (SELECT src3.*, src4.value as src4_value, src4.key as src4_key FROM src src4 JOIN (SELECT src2.*, src1.key as src1_key, src1.value as src1_value FROM src src1 JOIN src src2 ON src1.key = src2.key) src3 ON src3.src1_key = src4.key) src5; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/join23.q -- diff --git a/ql/src/test/queries/clientpositive/join23.q b/ql/src/test/queries/clientpositive/join23.q index d48941a..bb35103 100644 --- a/ql/src/test/queries/clientpositive/join23.q +++
[01/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master 24969a8de -> 86b678f50
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/garbage.q.out
--
diff --git a/ql/src/test/results/compiler/errors/garbage.q.out
b/ql/src/test/results/compiler/errors/garbage.q.out
index fdca829..1f70a5f 100644
--- a/ql/src/test/results/compiler/errors/garbage.q.out
+++ b/ql/src/test/results/compiler/errors/garbage.q.out
@@ -1 +1 @@
-Parse Error: line 1:0 cannot recognize input near 'this' 'is' 'totally'
\ No newline at end of file
+Parse Error: line 2:0 cannot recognize input near 'this' 'is' 'totally'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out
--
diff --git
a/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out
b/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out
index 1bf1354..e54bbb0 100644
--- a/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out
+++ b/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out
@@ -1,2 +1,2 @@
Semantic Exception:
-Line 2:23 Cannot insert into target table because column number/types are
different 'dest1': Table insclause-0 has 2 columns, but query has 3 columns.
\ No newline at end of file
+Line 3:23 Cannot insert into target table because column number/types are
different 'dest1': Table insclause-0 has 2 columns, but query has 3 columns.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/invalid_dot.q.out
--
diff --git a/ql/src/test/results/compiler/errors/invalid_dot.q.out
b/ql/src/test/results/compiler/errors/invalid_dot.q.out
index df24ac6..e197d03 100644
--- a/ql/src/test/results/compiler/errors/invalid_dot.q.out
+++ b/ql/src/test/results/compiler/errors/invalid_dot.q.out
@@ -1,2 +1,2 @@
Semantic Exception:
-Line 2:36 . Operator is only supported on struct or list of struct types
'member'
\ No newline at end of file
+Line 3:36 . Operator is only supported on struct or list of struct types
'member'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/invalid_function_param2.q.out
--
diff --git a/ql/src/test/results/compiler/errors/invalid_function_param2.q.out
b/ql/src/test/results/compiler/errors/invalid_function_param2.q.out
index 889b71c..60606f7 100644
--- a/ql/src/test/results/compiler/errors/invalid_function_param2.q.out
+++ b/ql/src/test/results/compiler/errors/invalid_function_param2.q.out
@@ -1,2 +1,2 @@
Semantic Exception:
-Line 2:36 Wrong arguments ''abc'': No matching method for class
org.apache.hadoop.hive.ql.udf.UDFSubstr with (string, string). Possible
choices: _FUNC_(binary, int) _FUNC_(binary, int, int) _FUNC_(string, int)
_FUNC_(string, int, int)
\ No newline at end of file
+Line 3:36 Wrong arguments ''abc'': No matching method for class
org.apache.hadoop.hive.ql.udf.UDFSubstr with (string, string). Possible
choices: _FUNC_(binary, int) _FUNC_(binary, int, int) _FUNC_(string, int)
_FUNC_(string, int, int)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/invalid_index.q.out
--
diff --git a/ql/src/test/results/compiler/errors/invalid_index.q.out
b/ql/src/test/results/compiler/errors/invalid_index.q.out
index e1909b9..983a531 100644
--- a/ql/src/test/results/compiler/errors/invalid_index.q.out
+++ b/ql/src/test/results/compiler/errors/invalid_index.q.out
@@ -1,2 +1,2 @@
Semantic Exception:
-Line 2:36 [] not valid on non-collection types '0': string
\ No newline at end of file
+Line 3:36 [] not valid on non-collection types '0': string
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/invalid_select.q.out
--
diff --git a/ql/src/test/results/compiler/errors/invalid_select.q.out
b/ql/src/test/results/compiler/errors/invalid_select.q.out
index 9e468d7..31ee094 100644
--- a/ql/src/test/results/compiler/errors/invalid_select.q.out
+++ b/ql/src/test/results/compiler/errors/invalid_select.q.out
@@ -1 +1 @@
-Parse Error: line 3:6 missing EOF at '(' near 'trim'
\ No newline at end of file
+Parse Error: line 4:6 missing EOF at '(' near 'trim'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/compiler/errors/missing_overwrite.q.out
---
[09/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapjoin_test_outer.q -- diff --git a/ql/src/test/queries/clientpositive/mapjoin_test_outer.q b/ql/src/test/queries/clientpositive/mapjoin_test_outer.q index e2998d7..96275ab 100644 --- a/ql/src/test/queries/clientpositive/mapjoin_test_outer.q +++ b/ql/src/test/queries/clientpositive/mapjoin_test_outer.q @@ -1,3 +1,4 @@ +--! qt:dataset:src1 set hive.auto.convert.join = false; -- SORT_QUERY_RESULTS http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce1.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce1.q b/ql/src/test/queries/clientpositive/mapreduce1.q index e2b314f..06c7c53 100644 --- a/ql/src/test/queries/clientpositive/mapreduce1.q +++ b/ql/src/test/queries/clientpositive/mapreduce1.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.explain.user=false; CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce2.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce2.q b/ql/src/test/queries/clientpositive/mapreduce2.q index 5a19b3c..dfb8ec3 100644 --- a/ql/src/test/queries/clientpositive/mapreduce2.q +++ b/ql/src/test/queries/clientpositive/mapreduce2.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.explain.user=false; CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce3.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce3.q b/ql/src/test/queries/clientpositive/mapreduce3.q index c31ede7..ad9908f 100644 --- a/ql/src/test/queries/clientpositive/mapreduce3.q +++ b/ql/src/test/queries/clientpositive/mapreduce3.q @@ -1,3 +1,4 @@ +--! qt:dataset:src CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce4.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce4.q b/ql/src/test/queries/clientpositive/mapreduce4.q index 0c15f1f..74e907e 100644 --- a/ql/src/test/queries/clientpositive/mapreduce4.q +++ b/ql/src/test/queries/clientpositive/mapreduce4.q @@ -1,3 +1,4 @@ +--! qt:dataset:src CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce5.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce5.q b/ql/src/test/queries/clientpositive/mapreduce5.q index d7c53d2..841ead5 100644 --- a/ql/src/test/queries/clientpositive/mapreduce5.q +++ b/ql/src/test/queries/clientpositive/mapreduce5.q @@ -1,3 +1,4 @@ +--! qt:dataset:src CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce6.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce6.q b/ql/src/test/queries/clientpositive/mapreduce6.q index c37181d..213db77 100644 --- a/ql/src/test/queries/clientpositive/mapreduce6.q +++ b/ql/src/test/queries/clientpositive/mapreduce6.q @@ -1,3 +1,4 @@ +--! qt:dataset:src CREATE TABLE dest1(key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce7.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce7.q b/ql/src/test/queries/clientpositive/mapreduce7.q index 0da41d2..77b82ef 100644 --- a/ql/src/test/queries/clientpositive/mapreduce7.q +++ b/ql/src/test/queries/clientpositive/mapreduce7.q @@ -1,3 +1,4 @@ +--! qt:dataset:src CREATE TABLE dest1(k STRING, v STRING, key INT, ten INT, one INT, value STRING) STORED AS TEXTFILE; EXPLAIN http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/mapreduce8.q -- diff --git a/ql/src/test/queries/clientpositive/mapreduce8.q b/ql/src/test/queries/clientpositive/mapreduce8.q index 35d9ed9..7db7510 100644 --- a/ql/src/test/queries/clientpositive/mapreduce8.q +++ b/ql/src/test/queries/clientpositive/mapreduce8.q @@ -1,3 +1,4 @@ +--
[18/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
--
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
b/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
index 6684fd3..ca7e6c7 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_custom_key.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
CREATE TABLE accumulo_ck_1(key struct,
value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
--
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
b/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
index 038633f..615e5f8 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_custom_key2.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
CREATE TABLE accumulo_ck_3(key struct,
value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES (
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_joins.q
--
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_joins.q
b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
index b72ec6b..a6b09ab 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_joins.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_joins.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
DROP TABLE users;
DROP TABLE states;
DROP TABLE countries;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
--
diff --git
a/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
b/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
index 0f064af..4cb93b5 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_predicate_pushdown.q
@@ -1,3 +1,5 @@
+--! qt:dataset:src
+--! qt:dataset:part
CREATE TABLE accumulo_pushdown(key string, value string)
STORED BY 'org.apache.hadoop.hive.accumulo.AccumuloStorageHandler'
WITH SERDEPROPERTIES ("accumulo.columns.mapping" = ":rowid,cf:string");
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_queries.q
--
diff --git a/accumulo-handler/src/test/queries/positive/accumulo_queries.q
b/accumulo-handler/src/test/queries/positive/accumulo_queries.q
index 0aceaa0..bc93d8a 100644
--- a/accumulo-handler/src/test/queries/positive/accumulo_queries.q
+++ b/accumulo-handler/src/test/queries/positive/accumulo_queries.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
-- remove these; after HIVE-18802 is fixed
set hive.optimize.index.filter=false;
set hive.optimize.ppd=false;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
--
diff --git
a/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
b/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
index f904d3f..c12f962 100644
---
a/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
+++
b/accumulo-handler/src/test/queries/positive/accumulo_single_sourced_multi_insert.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
-- HIVE-4375 Single sourced multi insert consists of native and non-native
table mixed throws NPE
CREATE TABLE src_x1(key string, value string);
CREATE TABLE src_x2(key string, value string)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
--
diff --git a/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
b/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
index 910ffda..9922092 100644
--- a/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
+++ b/contrib/src/test/queries/clientnegative/case_with_row_sequence.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
set hive.exec.submitviachild=true;
set hive.exec.submit.local.task.via.child=true;
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/contrib/src/test/queries/client
[03/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/windowing_udaf2.q
--
diff --git a/ql/src/test/queries/clientpositive/windowing_udaf2.q
b/ql/src/test/queries/clientpositive/windowing_udaf2.q
index b813657..e2a6e6d 100644
--- a/ql/src/test/queries/clientpositive/windowing_udaf2.q
+++ b/ql/src/test/queries/clientpositive/windowing_udaf2.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
-- user-added aggregates should be usable as windowing functions
create temporary function mysum as
'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/ambiguous_join_col.q
--
diff --git a/ql/src/test/queries/negative/ambiguous_join_col.q
b/ql/src/test/queries/negative/ambiguous_join_col.q
index e70aae4..ff4ba0f 100644
--- a/ql/src/test/queries/negative/ambiguous_join_col.q
+++ b/ql/src/test/queries/negative/ambiguous_join_col.q
@@ -1,2 +1,4 @@
+--! qt:dataset:src1
+--! qt:dataset:src
FROM src src1 JOIN src src2 ON src1.key = src2.key
INSERT OVERWRITE TABLE dest1 SELECT key
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/duplicate_alias.q
--
diff --git a/ql/src/test/queries/negative/duplicate_alias.q
b/ql/src/test/queries/negative/duplicate_alias.q
index 5fd2246..c3df285 100644
--- a/ql/src/test/queries/negative/duplicate_alias.q
+++ b/ql/src/test/queries/negative/duplicate_alias.q
@@ -1,2 +1,3 @@
+--! qt:dataset:src
FROM src a JOIN src a ON (a.key = a.key)
INSERT OVERWRITE TABLE dest1 SELECT a.key, a.value
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/garbage.q
--
diff --git a/ql/src/test/queries/negative/garbage.q
b/ql/src/test/queries/negative/garbage.q
index 6c8c751..93bb546 100644
--- a/ql/src/test/queries/negative/garbage.q
+++ b/ql/src/test/queries/negative/garbage.q
@@ -1 +1,2 @@
+--! qt:dataset:src
this is totally garbage SELECT src.key WHERE a lot of garbage
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/insert_wrong_number_columns.q
--
diff --git a/ql/src/test/queries/negative/insert_wrong_number_columns.q
b/ql/src/test/queries/negative/insert_wrong_number_columns.q
index aadfbde..a5919e7 100644
--- a/ql/src/test/queries/negative/insert_wrong_number_columns.q
+++ b/ql/src/test/queries/negative/insert_wrong_number_columns.q
@@ -1,2 +1,3 @@
+--! qt:dataset:src
FROM src
INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value, 1 WHERE src.key < 100
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/invalid_dot.q
--
diff --git a/ql/src/test/queries/negative/invalid_dot.q
b/ql/src/test/queries/negative/invalid_dot.q
index 36b9bd2..e13535a 100644
--- a/ql/src/test/queries/negative/invalid_dot.q
+++ b/ql/src/test/queries/negative/invalid_dot.q
@@ -1,2 +1,3 @@
+--! qt:dataset:src
FROM src
INSERT OVERWRITE TABLE dest1 SELECT src.value.member WHERE src.key < 100
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/invalid_function_param2.q
--
diff --git a/ql/src/test/queries/negative/invalid_function_param2.q
b/ql/src/test/queries/negative/invalid_function_param2.q
index 3543449..8212f9f 100644
--- a/ql/src/test/queries/negative/invalid_function_param2.q
+++ b/ql/src/test/queries/negative/invalid_function_param2.q
@@ -1,2 +1,3 @@
+--! qt:dataset:src
FROM src
INSERT OVERWRITE TABLE dest1 SELECT substr('1234', 'abc'), src.value WHERE
src.key < 100
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/invalid_index.q
--
diff --git a/ql/src/test/queries/negative/invalid_index.q
b/ql/src/test/queries/negative/invalid_index.q
index 146bc5d..dc36804 100644
--- a/ql/src/test/queries/negative/invalid_index.q
+++ b/ql/src/test/queries/negative/invalid_index.q
@@ -1,2 +1,3 @@
+--! qt:dataset:src
FROM src
INSERT OVERWRITE TABLE dest1 SELECT src.key[0], src.value
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/negative/invalid_select.q
--
diff --git a/ql/src/test/queries/negative/invalid_select.q
b/ql/src/test/queries/negative/invalid_select.q
index fd12985..8bb0982 100644
--- a/ql/src/test/queries/negative/invalid_select.q
+++ b/ql/src/test/queries/negative/invalid_select.q
@@ -1,3 +1,4 @@
+--! qt:dataset:src
SELECT
trim(trim(a))
[06/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_explain_groupbyshuffle.q -- diff --git a/ql/src/test/queries/clientpositive/spark_explain_groupbyshuffle.q b/ql/src/test/queries/clientpositive/spark_explain_groupbyshuffle.q index cd2cba1..c9dc56c 100644 --- a/ql/src/test/queries/clientpositive/spark_explain_groupbyshuffle.q +++ b/ql/src/test/queries/clientpositive/spark_explain_groupbyshuffle.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.spark.use.groupby.shuffle=true; explain select key, count(value) from src group by key; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_explainuser_1.q -- diff --git a/ql/src/test/queries/clientpositive/spark_explainuser_1.q b/ql/src/test/queries/clientpositive/spark_explainuser_1.q index 43252f0..92e6554 100644 --- a/ql/src/test/queries/clientpositive/spark_explainuser_1.q +++ b/ql/src/test/queries/clientpositive/spark_explainuser_1.q @@ -1,3 +1,12 @@ +--! qt:dataset:srcpart +--! qt:dataset:src_cbo +--! qt:dataset:src1 +--! qt:dataset:src +--! qt:dataset:part +--! qt:dataset:lineitem +--! qt:dataset:cbo_t3 +--! qt:dataset:cbo_t2 +--! qt:dataset:cbo_t1 set hive.strict.checks.bucketing=false; set hive.mapred.mode=nonstrict; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_local_queries.q -- diff --git a/ql/src/test/queries/clientpositive/spark_local_queries.q b/ql/src/test/queries/clientpositive/spark_local_queries.q index 8238113..e43f440 100644 --- a/ql/src/test/queries/clientpositive/spark_local_queries.q +++ b/ql/src/test/queries/clientpositive/spark_local_queries.q @@ -1,3 +1,6 @@ +--! qt:dataset:srcpart +--! qt:dataset:src1 +--! qt:dataset:src -- Some basic tests to test HoS works with spark.master = local -- Test that a basic explain plan can be generated http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_multi_insert_parallel_orderby.q -- diff --git a/ql/src/test/queries/clientpositive/spark_multi_insert_parallel_orderby.q b/ql/src/test/queries/clientpositive/spark_multi_insert_parallel_orderby.q index 2be0d52..bef1f85 100644 --- a/ql/src/test/queries/clientpositive/spark_multi_insert_parallel_orderby.q +++ b/ql/src/test/queries/clientpositive/spark_multi_insert_parallel_orderby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.exec.reducers.bytes.per.reducer=256; set hive.optimize.sampling.orderby=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_opt_shuffle_serde.q -- diff --git a/ql/src/test/queries/clientpositive/spark_opt_shuffle_serde.q b/ql/src/test/queries/clientpositive/spark_opt_shuffle_serde.q index 2c4691a..94d4d7a 100644 --- a/ql/src/test/queries/clientpositive/spark_opt_shuffle_serde.q +++ b/ql/src/test/queries/clientpositive/spark_opt_shuffle_serde.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.spark.optimize.shuffle.serde=true; set hive.spark.use.groupby.shuffle=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_union_merge.q -- diff --git a/ql/src/test/queries/clientpositive/spark_union_merge.q b/ql/src/test/queries/clientpositive/spark_union_merge.q index 3121078..6f2b059 100644 --- a/ql/src/test/queries/clientpositive/spark_union_merge.q +++ b/ql/src/test/queries/clientpositive/spark_union_merge.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.mapred.mode=nonstrict; -- union case: both subqueries are map jobs on same input, followed by filesink -- mostly copied from union.q http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_use_op_stats.q -- diff --git a/ql/src/test/queries/clientpositive/spark_use_op_stats.q b/ql/src/test/queries/clientpositive/spark_use_op_stats.q index b559bc0..8a7ce2c 100644 --- a/ql/src/test/queries/clientpositive/spark_use_op_stats.q +++ b/ql/src/test/queries/clientpositive/spark_use_op_stats.q @@ -1,3 +1,5 @@ +--! qt:dataset:src1 +--! qt:dataset:src set hive.mapred.mode=nonstrict; set hive.spark.use.op.stats=false; set hive.auto.convert.join=false; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientpositive/spark_use_ts_stats_for_mapjoin.q -- diff --git a/ql/src/test/queries/clientpositive/spark_use_ts_stats_for_mapjoin.q b/ql/src/test/quer
[02/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_map_values_arg_num.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_map_values_arg_num.q.out b/ql/src/test/results/clientnegative/udf_map_values_arg_num.q.out index f5deee4..6320a2e 100644 --- a/ql/src/test/results/clientnegative/udf_map_values_arg_num.q.out +++ b/ql/src/test/results/clientnegative/udf_map_values_arg_num.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10015]: Line 1:7 Arguments length mismatch '"2"': The function MAP_VALUES only accepts 1 argument. +FAILED: SemanticException [Error 10015]: Line 2:7 Arguments length mismatch '"2"': The function MAP_VALUES only accepts 1 argument. http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_map_values_arg_type.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_map_values_arg_type.q.out b/ql/src/test/results/clientnegative/udf_map_values_arg_type.q.out index affb070..0af2e65 100644 --- a/ql/src/test/results/clientnegative/udf_map_values_arg_type.q.out +++ b/ql/src/test/results/clientnegative/udf_map_values_arg_type.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 1:18 Argument type mismatch '4': "map" is expected at function MAP_VALUES, but "array" is found +FAILED: SemanticException [Error 10016]: Line 2:18 Argument type mismatch '4': "map" is expected at function MAP_VALUES, but "array" is found http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_printf_wrong1.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_printf_wrong1.q.out b/ql/src/test/results/clientnegative/udf_printf_wrong1.q.out index 41359d0..298b8a9 100644 --- a/ql/src/test/results/clientnegative/udf_printf_wrong1.q.out +++ b/ql/src/test/results/clientnegative/udf_printf_wrong1.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10015]: Line 2:7 Arguments length mismatch 'printf': The function PRINTF(String format, Obj... args) needs at least one arguments. +FAILED: SemanticException [Error 10015]: Line 3:7 Arguments length mismatch 'printf': The function PRINTF(String format, Obj... args) needs at least one arguments. http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_printf_wrong2.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_printf_wrong2.q.out b/ql/src/test/results/clientnegative/udf_printf_wrong2.q.out index 6a7c8e6..09e7b84 100644 --- a/ql/src/test/results/clientnegative/udf_printf_wrong2.q.out +++ b/ql/src/test/results/clientnegative/udf_printf_wrong2.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 2:14 Argument type mismatch '100': Argument 1 of function PRINTF must be "string", but "int" was found. +FAILED: SemanticException [Error 10016]: Line 3:14 Argument type mismatch '100': Argument 1 of function PRINTF must be "string", but "int" was found. http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_printf_wrong3.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_printf_wrong3.q.out b/ql/src/test/results/clientnegative/udf_printf_wrong3.q.out index c020fda..95063b9 100644 --- a/ql/src/test/results/clientnegative/udf_printf_wrong3.q.out +++ b/ql/src/test/results/clientnegative/udf_printf_wrong3.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 2:32 Argument type mismatch '"argument"': Argument 2 of function PRINTF must be "PRIMITIVE", but "array" was found. +FAILED: SemanticException [Error 10016]: Line 3:32 Argument type mismatch '"argument"': Argument 2 of function PRINTF must be "PRIMITIVE", but "array" was found. http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_printf_wrong4.q.out -- diff --git a/ql/src/test/results/clientnegative/udf_printf_wrong4.q.out b/ql/src/test/results/clientnegative/udf_printf_wrong4.q.out index c020fda..95063b9 100644 --- a/ql/src/test/results/clientnegative/udf_printf_wrong4.q.out +++ b/ql/src/test/results/clientnegative/udf_printf_wrong4.q.out @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 2:32 Argument type mismatch '"argument"': Argument 2 of function PRINTF must be "PRIMITIVE", but "array" was found. +FAILED: SemanticException [Error 10016]: Line 3:32 Argument type mismatch '"argument"': Argument 2 of function PRINTF must be "PRIMITIVE", but "array" was found. http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/results/clientnegative/udf_size_wrong_args_len.q.o
[17/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
--
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index c33851f..574b822 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -146,6 +146,7 @@ public class QTestUtil {
private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically
target
public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables";
+ public static final String TEST_HIVE_USER_PROPERTY = "test.hive.user";
private String testWarehouse;
private final String testFiles;
@@ -206,7 +207,7 @@ public class QTestUtil {
getSrcTables().add(table);
storeSrcTables();
}
-
+
public static Set initSrcTables() {
if (srcTables == null){
initSrcTablesFromSystemProperty();
@@ -224,18 +225,12 @@ public class QTestUtil {
srcTables = new HashSet();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
-String defaultTestSrcTables =
"src,src1,srcbucket,srcbucket2,src_json,src_thrift," +
-
"src_sequencefile,srcpart,alltypesorc,src_hbase,cbo_t1,cbo_t2,cbo_t3,src_cbo,part,"
+
-"lineitem,alltypesparquet";
-for (String srcTable : System.getProperty(TEST_SRC_TABLES_PROPERTY,
defaultTestSrcTables).trim().split(",")) {
+for (String srcTable : System.getProperty(TEST_SRC_TABLES_PROPERTY,
"").trim().split(",")) {
srcTable = srcTable.trim();
if (!srcTable.isEmpty()) {
srcTables.add(srcTable);
}
}
-if (srcTables.isEmpty()) {
- throw new RuntimeException("Source tables cannot be empty");
-}
}
private CliDriver getCliDriver() {
@@ -1070,6 +1065,16 @@ public class QTestUtil {
clearUDFsCreatedDuringTests();
clearKeysCreatedInTests();
}
+
+ protected void clearSettingsCreatedInTests() throws IOException {
+getCliDriver().processLine(String.format("set
hive.security.authorization.enabled=false;"));
+getCliDriver().processLine(String.format("set user.name=%s;",
+System.getProperty(TEST_HIVE_USER_PROPERTY, "hive_test_user")));
+
+getCliDriver().processLine("set
hive.metastore.partition.name.whitelist.pattern=;");
+getCliDriver().processLine("set hive.test.mode=false;");
+getCliDriver().processLine("set hive.mapred.mode=nonstrict;");
+ }
protected void initConfFromSetup() throws Exception {
setup.preTest(conf);
@@ -1091,7 +1096,7 @@ public class QTestUtil {
clearTablesCreatedDuringTests();
clearUDFsCreatedDuringTests();
clearKeysCreatedInTests();
-
+
cleanupFromFile();
// delete any contents in the warehouse dir
@@ -1199,7 +1204,7 @@ public class QTestUtil {
}
}
- private void initDataset(String table) {
+ protected synchronized void initDataset(String table) {
if (getSrcTables().contains(table)){
return;
}
@@ -1262,6 +1267,7 @@ public class QTestUtil {
createSources(fileName);
}
+clearSettingsCreatedInTests();
initDataSetForTest(file);
HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index f087393..a071689 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.io.Serializable;
import java.util.List;
@@ -104,6 +105,9 @@ public class CoreParseNegative extends CliAdapter{
qt.init(fname);
firstRun = false;
}
+
+ qt.cliInit(new File(fpath), false);
+
ASTNode tree = qt.parseQuery(fname);
List> tasks = qt.analyzeAST(tree);
fail("Unexpected success for query: " + fname + debugHint);
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
--
diff --git
a/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
b/itests/util/src/main/java/org/apache/hive/beeline/QFileBeeLineClient.java
index e6bf54b..be4c6e8
[15/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_exists_implicit_gby.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_exists_implicit_gby.q b/ql/src/test/queries/clientnegative/subquery_exists_implicit_gby.q index 497f3c2..6650dc2 100644 --- a/ql/src/test/queries/clientnegative/subquery_exists_implicit_gby.q +++ b/ql/src/test/queries/clientnegative/subquery_exists_implicit_gby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src select * http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_in_groupby.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_in_groupby.q b/ql/src/test/queries/clientnegative/subquery_in_groupby.q index a9bc6ee..74e9a07 100644 --- a/ql/src/test/queries/clientnegative/subquery_in_groupby.q +++ b/ql/src/test/queries/clientnegative/subquery_in_groupby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src select count(*) http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_in_lhs.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_in_lhs.q b/ql/src/test/queries/clientnegative/subquery_in_lhs.q index 087c3f3..e24de1e 100644 --- a/ql/src/test/queries/clientnegative/subquery_in_lhs.q +++ b/ql/src/test/queries/clientnegative/subquery_in_lhs.q @@ -1,2 +1,3 @@ +--! qt:dataset:part select * from part where (select max(p_size) from part) IN (select p_size from part); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_in_on.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_in_on.q b/ql/src/test/queries/clientnegative/subquery_in_on.q index 5f44f72..fb30d44 100644 --- a/ql/src/test/queries/clientnegative/subquery_in_on.q +++ b/ql/src/test/queries/clientnegative/subquery_in_on.q @@ -1,3 +1,4 @@ +--! qt:dataset:part -- subquery in ON clause explain SELECT p1.p_name FROM part p1 LEFT JOIN (select p_type as p_col from part ) p2 ON (select pp1.p_type as p_col from part pp1 where pp1.p_partkey = p2.p_col); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_multiple_cols_in_select.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_multiple_cols_in_select.q b/ql/src/test/queries/clientnegative/subquery_multiple_cols_in_select.q index 6805c5b..c7a53e0 100644 --- a/ql/src/test/queries/clientnegative/subquery_multiple_cols_in_select.q +++ b/ql/src/test/queries/clientnegative/subquery_multiple_cols_in_select.q @@ -1,3 +1,4 @@ +--! qt:dataset:src explain http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_notexists_implicit_gby.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_notexists_implicit_gby.q b/ql/src/test/queries/clientnegative/subquery_notexists_implicit_gby.q index 57c72d2..63847bf 100644 --- a/ql/src/test/queries/clientnegative/subquery_notexists_implicit_gby.q +++ b/ql/src/test/queries/clientnegative/subquery_notexists_implicit_gby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src select * http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_scalar_corr_multi_rows.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_scalar_corr_multi_rows.q b/ql/src/test/queries/clientnegative/subquery_scalar_corr_multi_rows.q index e71a60d..47fb513 100644 --- a/ql/src/test/queries/clientnegative/subquery_scalar_corr_multi_rows.q +++ b/ql/src/test/queries/clientnegative/subquery_scalar_corr_multi_rows.q @@ -1,3 +1,4 @@ +--! qt:dataset:part -- inner query produces more than one row select * from part where p_size > (select count(*) from part p where p.p_mfgr = part.p_mfgr group by p_type); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/subquery_scalar_multi_columns.q -- diff --git a/ql/src/test/queries/clientnegative/subquery_scalar_multi_columns.q b/ql/src/test/queries/clientnegative/subquery_scalar_multi_columns.q index 1f6ad1a..948ed0e 100644 --- a/ql/src/test/queries/clientnegative/subquery_scalar_multi_columns.q +++ b/ql/src/test/queries/clientnegative/subquery_scalar_multi_columns.q @@ -1,2 +1,3 @@ +--! qt:dataset:part set hive.mapred.mode=nonstrict; select p_name from part where p_size > (select p_size, p_
[16/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby2_multi_distinct.q -- diff --git a/ql/src/test/queries/clientnegative/groupby2_multi_distinct.q b/ql/src/test/queries/clientnegative/groupby2_multi_distinct.q index e3b0066..0626da1 100644 --- a/ql/src/test/queries/clientnegative/groupby2_multi_distinct.q +++ b/ql/src/test/queries/clientnegative/groupby2_multi_distinct.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.map.aggr=false; set hive.groupby.skewindata=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby3_map_skew_multi_distinct.q -- diff --git a/ql/src/test/queries/clientnegative/groupby3_map_skew_multi_distinct.q b/ql/src/test/queries/clientnegative/groupby3_map_skew_multi_distinct.q index 168aeb1..51c22be 100644 --- a/ql/src/test/queries/clientnegative/groupby3_map_skew_multi_distinct.q +++ b/ql/src/test/queries/clientnegative/groupby3_map_skew_multi_distinct.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.map.aggr=true; set hive.groupby.skewindata=true; set mapred.reduce.tasks=31; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby3_multi_distinct.q -- diff --git a/ql/src/test/queries/clientnegative/groupby3_multi_distinct.q b/ql/src/test/queries/clientnegative/groupby3_multi_distinct.q index 1a28477..1304985 100644 --- a/ql/src/test/queries/clientnegative/groupby3_multi_distinct.q +++ b/ql/src/test/queries/clientnegative/groupby3_multi_distinct.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.map.aggr=false; set hive.groupby.skewindata=true; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_cube1.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_cube1.q b/ql/src/test/queries/clientnegative/groupby_cube1.q index a0bc177..16d910b 100644 --- a/ql/src/test/queries/clientnegative/groupby_cube1.q +++ b/ql/src/test/queries/clientnegative/groupby_cube1.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.map.aggr=false; SELECT key, count(distinct value) FROM src GROUP BY key with cube; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_cube2.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_cube2.q b/ql/src/test/queries/clientnegative/groupby_cube2.q index f8ecb6a..34a02aa 100644 --- a/ql/src/test/queries/clientnegative/groupby_cube2.q +++ b/ql/src/test/queries/clientnegative/groupby_cube2.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.map.aggr=true; SELECT key, value, count(distinct value) FROM src GROUP BY key, value with cube; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q b/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q index cddbe1a..e0d8ef4 100644 --- a/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q +++ b/ql/src/test/queries/clientnegative/groupby_cube_multi_gby.q @@ -1,3 +1,4 @@ +--! qt:dataset:src create table t1 like src; create table t2 like src; http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_invalid_position.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_invalid_position.q b/ql/src/test/queries/clientnegative/groupby_invalid_position.q index 173a752..55536c6 100644 --- a/ql/src/test/queries/clientnegative/groupby_invalid_position.q +++ b/ql/src/test/queries/clientnegative/groupby_invalid_position.q @@ -1,3 +1,4 @@ +--! qt:dataset:src set hive.groupby.orderby.position.alias=true; -- invalid position alias in group by http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_key.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_key.q b/ql/src/test/queries/clientnegative/groupby_key.q index 2097015..eb529e8 100644 --- a/ql/src/test/queries/clientnegative/groupby_key.q +++ b/ql/src/test/queries/clientnegative/groupby_key.q @@ -1 +1,2 @@ +--! qt:dataset:src SELECT concat(value, concat(value)) FROM src GROUP BY concat(value); http://git-wip-us.apache.org/repos/asf/hive/blob/86b678f5/ql/src/test/queries/clientnegative/groupby_rollup1.q -- diff --git a/ql/src/test/queries/clientnegative/groupby_rollup1.q
[19/19] hive git commit: HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich)
HIVE-18862: qfiles: prepare .q files for using datasets (Laszlo Bodor via Zoltan Haindrich) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/86b678f5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/86b678f5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/86b678f5 Branch: refs/heads/master Commit: 86b678f502bc6bb86c2279ef1288627bd8e7421a Parents: 24969a8 Author: Laszlo Bodor Authored: Sat Apr 14 08:32:13 2018 +0200 Committer: Zoltan Haindrich Committed: Sat Apr 14 08:32:13 2018 +0200 -- .../test/queries/positive/accumulo_custom_key.q | 1 + .../queries/positive/accumulo_custom_key2.q | 1 + .../src/test/queries/positive/accumulo_joins.q | 1 + .../positive/accumulo_predicate_pushdown.q | 2 + .../test/queries/positive/accumulo_queries.q| 1 + .../accumulo_single_sourced_multi_insert.q | 1 + .../clientnegative/case_with_row_sequence.q | 1 + .../clientnegative/invalid_row_sequence.q | 1 + .../test/queries/clientnegative/udtf_explode2.q | 1 + .../src/test/queries/clientpositive/dboutput.q | 1 + .../queries/clientpositive/fileformat_base64.q | 1 + .../queries/clientpositive/java_mr_example.q| 1 + .../clientpositive/lateral_view_explode2.q | 1 + .../queries/clientpositive/serde_typedbytes.q | 1 + .../queries/clientpositive/serde_typedbytes2.q | 1 + .../queries/clientpositive/serde_typedbytes3.q | 1 + .../queries/clientpositive/serde_typedbytes4.q | 1 + .../queries/clientpositive/serde_typedbytes5.q | 1 + .../clientpositive/serde_typedbytes_null.q | 1 + .../queries/clientpositive/udaf_example_avg.q | 1 + .../clientpositive/udaf_example_group_concat.q | 1 + .../queries/clientpositive/udaf_example_max.q | 1 + .../queries/clientpositive/udaf_example_max_n.q | 1 + .../queries/clientpositive/udaf_example_min.q | 1 + .../queries/clientpositive/udaf_example_min_n.q | 1 + .../queries/clientpositive/udf_example_add.q| 1 + .../clientpositive/udf_example_arraymapstruct.q | 1 + .../queries/clientpositive/udf_example_format.q | 1 + .../queries/clientpositive/udf_row_sequence.q | 1 + .../test/queries/clientpositive/udtf_explode2.q | 1 + .../clientpositive/udtf_output_on_close.q | 1 + .../src/test/queries/clientpositive/url_hook.q | 1 + data/files/datasets/alltypesorc/load.hive.sql | 21 ++ .../datasets/alltypesparquet/load.hive.sql | 39 +++ data/files/datasets/cbo_t1/load.hive.sql| 8 + data/files/datasets/cbo_t2/load.hive.sql| 8 + data/files/datasets/cbo_t3/load.hive.sql| 8 + data/files/datasets/lineitem/load.hive.sql | 23 ++ data/files/datasets/part/load.hive.sql | 16 + data/files/datasets/src/load.hive.sql | 7 + data/files/datasets/src1/load.hive.sql | 7 + data/files/datasets/src_cbo/load.hive.sql | 8 + data/files/datasets/src_hbase/load.hive.sql | 14 + data/files/datasets/src_json/load.hive.sql | 7 + .../datasets/src_sequencefile/load.hive.sql | 7 + data/files/datasets/src_thrift/load.hive.sql| 10 + data/files/datasets/srcbucket/load.hive.sql | 13 + data/files/datasets/srcbucket2/load.hive.sql| 15 + data/files/datasets/srcpart/load.hive.sql | 19 ++ data/scripts/q_test_druid_init.sql | 29 -- data/scripts/q_test_init.sql| 293 -- data/scripts/q_test_init_compare.sql| 26 -- data/scripts/q_test_init_contrib.sql| 29 -- data/scripts/q_test_init_for_minimr.sql | 48 --- data/scripts/q_test_init_src.sql| 9 - data/scripts/q_test_init_src_with_stats.sql | 9 - data/scripts/q_test_init_tez.sql| 78 - .../generatehfiles_require_family_path.q| 1 + .../test/queries/positive/external_table_ppd.q | 1 + .../queries/positive/hbase_binary_map_queries.q | 1 + .../positive/hbase_binary_map_queries_prefix.q | 1 + .../positive/hbase_binary_storage_queries.q | 1 + .../src/test/queries/positive/hbase_bulk.q | 1 + .../test/queries/positive/hbase_custom_key.q| 1 + .../test/queries/positive/hbase_custom_key2.q | 1 + .../test/queries/positive/hbase_custom_key3.q | 1 + .../src/test/queries/positive/hbase_ddl.q | 1 + .../test/queries/positive/hbase_handler_bulk.q | 1 + .../queries/positive/hbase_handler_snapshot.q | 1 + .../src/test/queries/positive/hbase_joins.q | 1 + .../src/test/queries/positive/hbase_ppd_join.q | 1 + .../test/queries/positive/hbase_ppd_key_range.q | 2 + .../src/test/queries/positive/hbase_pushdown.q | 2 + .../src/test/queries/positive/hbase_queries.q | 1 + .../test/queries/positive/hbase_scan_params.q | 1 + .../hbase_
[2/2] hive git commit: HIVE-18926: Imporve operator-tree matching (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-18926: Imporve operator-tree matching (Zoltan Haindrich reviewed by
Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2394c5b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2394c5b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2394c5b
Branch: refs/heads/master
Commit: a2394c5bf2e5b257a3a1b3e2bda4a25b92005315
Parents: 1e884cc
Author: Zoltan Haindrich
Authored: Tue Mar 27 13:00:44 2018 +0200
Committer: Zoltan Haindrich
Committed: Tue Mar 27 13:00:44 2018 +0200
--
.../test/resources/testconfiguration.properties | 1 +
.../java/org/apache/hadoop/hive/ql/Context.java | 10 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 29 +-
.../java/org/apache/hadoop/hive/ql/IDriver.java | 2 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 3 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 34 +-
.../ql/optimizer/signature/OpSignature.java | 24 +-
.../ql/optimizer/signature/OpTreeSignature.java | 33 +-
.../hive/ql/optimizer/signature/Signature.java | 4 +-
.../ql/optimizer/signature/SignatureUtils.java | 8 +-
.../stats/annotation/StatsRulesProcFactory.java | 19 +-
.../hive/ql/plan/CommonMergeJoinDesc.java | 2 -
.../hadoop/hive/ql/plan/JoinCondDesc.java | 6 +-
.../hive/ql/plan/LateralViewJoinDesc.java | 2 -
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 5 -
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 7 +-
.../hive/ql/plan/mapper/EmptyStatsSource.java | 10 +
.../hive/ql/plan/mapper/GroupTransformer.java | 4 +-
.../hadoop/hive/ql/plan/mapper/PlanMapper.java | 79 -
.../hive/ql/plan/mapper/PlanMapperProcess.java | 47 ---
.../hive/ql/plan/mapper/RuntimeStatsSource.java | 29 --
.../plan/mapper/SimpleRuntimeStatsSource.java | 10 +-
.../hadoop/hive/ql/plan/mapper/StatsSource.java | 7 +
.../hadoop/hive/ql/reexec/ReExecDriver.java | 14 +-
.../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 2 +-
.../signature/TestOperatorSignature.java| 9 +-
.../ql/plan/mapping/TestCounterMapping.java | 8 +-
.../hive/ql/plan/mapping/TestOperatorCmp.java | 43 ++-
.../ql/plan/mapping/TestReOptimization.java | 6 +-
.../queries/clientpositive/explain_outputs.q| 15 +
.../beeline/explain_outputs.q.out | 332 ++
.../clientpositive/explain_outputs.q.out| 338 +++
.../llap/retry_failure_stat_changes.q.out | 8 +-
.../service/cli/operation/SQLOperation.java | 38 +--
34 files changed, 960 insertions(+), 228 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/a2394c5b/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 0d6aaae..5985dcf 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -901,6 +901,7 @@ beeline.positive.include=create_merge_compressed.q,\
colstats_all_nulls.q,\
drop_with_concurrency.q,\
escape_comments.q,\
+ explain_outputs.q,\
insert_overwrite_local_directory_1.q,\
mapjoin2.q,\
materialized_view_create_rewrite.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/a2394c5b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 58fa5f2..9ca8b00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -61,7 +60,6 @@ import org.apache.hadoop.hive.ql.parse.QB;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.mapper.EmptyStatsSource;
import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
-import org.apache.hadoop.hive.ql.plan.mapper.RuntimeStatsSource;
import org.apache.hadoop.hive.ql.plan.mapper.StatsSource;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.wm.WmContext;
@@ -161,7 +159,7 @@ public class Context {
private boolean isExplainPlan = false;
private PlanMapper planMapper = new PlanMapper();
- private RuntimeStatsSource runtimeStatsSource;
+ private StatsSource runtimeStatsSource;
private int executionIndex;
public void setOperation(Ope
[1/2] hive git commit: HIVE-18926: Imporve operator-tree matching (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 1e884cc5f -> a2394c5bf
http://git-wip-us.apache.org/repos/asf/hive/blob/a2394c5b/ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out
--
diff --git
a/ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out
b/ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out
index 9b58ce0..3b0d7e5 100644
--- a/ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out
+++ b/ql/src/test/results/clientpositive/llap/retry_failure_stat_changes.q.out
@@ -64,15 +64,15 @@ Stage-0
Stage-1
Reducer 3 llap
File Output Operator [FS_15]
-Group By Operator [GBY_13] (rows=1 width=8)
+Group By Operator [GBY_13] (runtime: rows=1 width=8)
Output:["_col0"],aggregations:["sum(VALUE._col0)"]
<-Reducer 2 [CUSTOM_SIMPLE_EDGE] llap
PARTITION_ONLY_SHUFFLE [RS_12]
-Group By Operator [GBY_11] (rows=1 width=8)
+Group By Operator [GBY_11] (runtime: rows=1 width=8)
Output:["_col0"],aggregations:["sum(_col0)"]
- Select Operator [SEL_9] (rows=15 width=8)
+ Select Operator [SEL_9] (runtime: rows=3 width=8)
Output:["_col0"]
-Merge Join Operator [MERGEJOIN_20] (rows=15 width=8)
+Merge Join Operator [MERGEJOIN_20] (runtime: rows=3 width=8)
Conds:RS_6._col0=RS_7._col0(Inner),Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE] llap
SHUFFLE [RS_6]
http://git-wip-us.apache.org/repos/asf/hive/blob/a2394c5b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
--
diff --git
a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index 5865abe..5fc935b 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -21,7 +21,6 @@ package org.apache.hive.service.cli.operation;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.security.PrivilegedExceptionAction;
import java.sql.SQLException;
@@ -53,9 +52,7 @@ import org.apache.hadoop.hive.ql.IDriver;
import org.apache.hadoop.hive.ql.QueryDisplay;
import org.apache.hadoop.hive.ql.QueryInfo;
import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.ExplainTask;
import org.apache.hadoop.hive.ql.exec.FetchTask;
-import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -90,8 +87,7 @@ import org.codehaus.jackson.map.ObjectMapper;
public class SQLOperation extends ExecuteStatementOperation {
private IDriver driver = null;
private CommandProcessorResponse response;
- private TableSchema resultSchema = null;
- private Schema mResultSchema = null;
+ private TableSchema resultSchema;
private AbstractSerDe serde = null;
private boolean fetchStarted = false;
private volatile MetricsScope currentSQLStateScope;
@@ -200,30 +196,7 @@ public class SQLOperation extends
ExecuteStatementOperation {
throw toSQLException("Error while compiling statement", response);
}
- mResultSchema = driver.getSchema();
-
- // hasResultSet should be true only if the query has a FetchTask
- // "explain" is an exception for now
- if(driver.getPlan().getFetchTask() != null) {
-//Schema has to be set
-if (mResultSchema == null || !mResultSchema.isSetFieldSchemas()) {
- throw new HiveSQLException("Error compiling query: Schema and
FieldSchema " +
- "should be set when query plan has a FetchTask");
-}
-resultSchema = new TableSchema(mResultSchema);
-setHasResultSet(true);
- } else {
-setHasResultSet(false);
- }
- // Set hasResultSet true if the plan has ExplainTask
- // TODO explain should use a FetchTask for reading
- for (Task task: driver.getPlan().getRootTasks())
{
-if (task.getClass() == ExplainTask.class) {
- resultSchema = new TableSchema(mResultSchema);
- setHasResultSet(true);
- break;
-}
- }
+ setHasResultSet(driver.hasResultSet());
} catch (HiveSQLException e) {
setState(OperationState.ERROR);
throw e;
@@ -447,8 +420,7 @@ public class SQLOperation extends ExecuteStatementOperation
{
public TableSchema getResultSetSchema() throws HiveSQLException {
// Since compilation is always a blocking RPC
hive git commit: HIVE-18965: HIVE-17990 didn't update derby SQL scripts (Alan Gates via Vineet Garg, Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master adc9ae722 -> e6daf43f0
HIVE-18965: HIVE-17990 didn't update derby SQL scripts (Alan Gates via Vineet
Garg, Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e6daf43f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e6daf43f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e6daf43f
Branch: refs/heads/master
Commit: e6daf43f0c41ff91014ae9769ba8ad8cab4e5920
Parents: adc9ae7
Author: Alan Gates
Authored: Wed Mar 21 14:34:12 2018 +0100
Committer: Zoltan Haindrich
Committed: Wed Mar 21 14:34:12 2018 +0100
--
.../upgrade/derby/052-HIVE-18965.derby.sql | 33
.../upgrade/derby/hive-schema-3.0.0.derby.sql | 31 +-
.../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 +
3 files changed, 64 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/e6daf43f/metastore/scripts/upgrade/derby/052-HIVE-18965.derby.sql
--
diff --git a/metastore/scripts/upgrade/derby/052-HIVE-18965.derby.sql
b/metastore/scripts/upgrade/derby/052-HIVE-18965.derby.sql
new file mode 100644
index 000..d6ec555
--- /dev/null
+++ b/metastore/scripts/upgrade/derby/052-HIVE-18965.derby.sql
@@ -0,0 +1,33 @@
+-- Upgrades for Schema Registry objects
+ALTER TABLE "APP"."SERDES" ADD COLUMN "DESCRIPTION" VARCHAR(4000);
+ALTER TABLE "APP"."SERDES" ADD COLUMN "SERIALIZER_CLASS" VARCHAR(4000);
+ALTER TABLE "APP"."SERDES" ADD COLUMN "DESERIALIZER_CLASS" VARCHAR(4000);
+ALTER TABLE "APP"."SERDES" ADD COLUMN "SERDE_TYPE" INTEGER;
+
+CREATE TABLE "APP"."I_SCHEMA" (
+ "SCHEMA_ID" bigint primary key,
+ "SCHEMA_TYPE" integer not null,
+ "NAME" varchar(256) unique,
+ "DB_ID" bigint references "APP"."DBS" ("DB_ID"),
+ "COMPATIBILITY" integer not null,
+ "VALIDATION_LEVEL" integer not null,
+ "CAN_EVOLVE" char(1) not null,
+ "SCHEMA_GROUP" varchar(256),
+ "DESCRIPTION" varchar(4000)
+);
+
+CREATE TABLE "APP"."SCHEMA_VERSION" (
+ "SCHEMA_VERSION_ID" bigint primary key,
+ "SCHEMA_ID" bigint references "APP"."I_SCHEMA" ("SCHEMA_ID"),
+ "VERSION" integer not null,
+ "CREATED_AT" bigint not null,
+ "CD_ID" bigint references "APP"."CDS" ("CD_ID"),
+ "STATE" integer not null,
+ "DESCRIPTION" varchar(4000),
+ "SCHEMA_TEXT" clob,
+ "FINGERPRINT" varchar(256),
+ "SCHEMA_VERSION_NAME" varchar(256),
+ "SERDE_ID" bigint references "APP"."SERDES" ("SERDE_ID")
+);
+
+CREATE UNIQUE INDEX "APP"."UNIQUE_SCHEMA_VERSION" ON "APP"."SCHEMA_VERSION"
("SCHEMA_ID", "VERSION");
http://git-wip-us.apache.org/repos/asf/hive/blob/e6daf43f/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql
--
diff --git a/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql
b/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql
index e335b84..2b1dd5b 100644
--- a/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-3.0.0.derby.sql
@@ -42,7 +42,7 @@ CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT
NULL, "PARAM_KEY" VARCH
CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME"
INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME"
VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT);
-CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128),
"SLIB" VARCHAR(4000));
+CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128),
"SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS"
VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER);
CREATE TABLE "APP"."PART_PRIVS" ("PART_GRANT_ID" BIGINT NOT NULL,
"CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR"
VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME"
VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_PRIV" VARCHAR(128));
@@ -392,6 +392,35 @@ ALTER TABLE "APP"."IDXS" ADD CONSTRAINT
"SQL110318025504980" CHECK (DEFERRED_REB
ALTER TABLE "APP"."SDS" ADD CONSTRAINT "SQL110318025505550" CHECK
(IS_COMPRESSED IN ('Y','N'));
+CREATE TABLE "APP"."I_SCHEMA" (
+ "SCHEMA_ID" bigint primary key,
+ "SCHEMA_TYPE" integer not null,
+ "NAME" varchar(256) unique,
+ "DB_ID" bigint references "APP"."DBS" ("DB_ID"),
+ "COMPATIBILITY" integer not null,
+ "VALIDATION_LEVEL" integer not null,
+ "CAN_EVOLVE" char(1) not null,
+ "SCHEMA_GROUP" varchar(256),
+ "DESCRIPTION" varchar(4000)
+);
+
+CREATE TABLE "APP"."SCHEMA_VERSION" (
+ "SCHEMA_VERSION_ID" bigint primary key,
+ "SCHEMA_ID" bigint references "APP"."I_SCHEMA" ("SCHEMA_ID"),
+ "VERSION" integer not null,
+ "CREATED_AT"
hive git commit: HIVE-18858: System properties in job configuration not resolved when submitting MR job (Daniel Voros via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master a16e8e9e4 -> adc9ae722
HIVE-18858: System properties in job configuration not resolved when submitting
MR job (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/adc9ae72
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/adc9ae72
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/adc9ae72
Branch: refs/heads/master
Commit: adc9ae722c5cec5883fa848ea8e87f5b50b988db
Parents: a16e8e9
Author: Daniel Voros
Authored: Wed Mar 21 08:00:18 2018 +0100
Committer: Zoltan Haindrich
Committed: Wed Mar 21 08:00:18 2018 +0100
--
.../java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java | 8
1 file changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/adc9ae72/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index e7fe4a2..7ff8ddc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -27,7 +27,9 @@ import java.lang.management.MemoryMXBean;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
@@ -173,6 +175,12 @@ public class ExecDriver extends Task
implements Serializable, Hadoop
CompilationOpContext opContext) {
super.initialize(queryState, queryPlan, driverContext, opContext);
+Iterator> iter = conf.iterator();
+while(iter.hasNext()) {
+ String key = iter.next().getKey();
+ conf.set(key, conf.get(key));
+}
+
job = new JobConf(conf, ExecDriver.class);
initializeFiles("tmpjars", getResource(conf,
SessionState.ResourceType.JAR));
[1/2] hive git commit: HIVE-18716: Delete unnecessary parameters from TaskFactory (Gergely Hajós via Zoltan Haindrich, Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master d2d50e694 -> 94152c997
http://git-wip-us.apache.org/repos/asf/hive/blob/94152c99/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index b7fbea4..9a91e3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -417,14 +417,14 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
} else {
CopyWork cw = new CopyWork(dataPath, destPath, false);
cw.setSkipSourceMmDirs(isSourceMm);
- copyTask = TaskFactory.get(cw, x.getConf());
+ copyTask = TaskFactory.get(cw);
}
LoadTableDesc loadTableWork = new LoadTableDesc(
loadPath, Utilities.getTableDesc(table), new TreeMap<>(), lft,
writeId);
loadTableWork.setStmtId(stmtId);
MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork,
null, false);
-Task loadTableTask = TaskFactory.get(mv, x.getConf());
+Task loadTableTask = TaskFactory.get(mv);
copyTask.addDependentTask(loadTableTask);
x.getTasks().add(copyTask);
return loadTableTask;
@@ -470,7 +470,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
x.getInputs(),
x.getOutputs(),
addPartitionDesc
-), x.getConf());
+));
}
private static Task addSinglePartition(URI fromURI, FileSystem fs,
ImportTableDesc tblDesc,
@@ -485,7 +485,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
// addPartitionDesc already has the right partition location
@SuppressWarnings("unchecked")
Task addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
- x.getOutputs(), addPartitionDesc), x.getConf());
+ x.getOutputs(), addPartitionDesc));
return addPartTask;
} else {
String srcLocation = partSpec.getLocation();
@@ -515,11 +515,11 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
} else {
CopyWork cw = new CopyWork(new Path(srcLocation), destPath, false);
cw.setSkipSourceMmDirs(isSourceMm);
-copyTask = TaskFactory.get(cw, x.getConf());
+copyTask = TaskFactory.get(cw);
}
Task addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
- x.getOutputs(), addPartitionDesc), x.getConf());
+ x.getOutputs(), addPartitionDesc));
// Note: this sets LoadFileType incorrectly for ACID; is that relevant
for import?
// See setLoadFileType and setIsAcidIow calls elsewhere for an
example.
LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc,
Utilities.getTableDesc(table),
@@ -529,7 +529,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
loadTableWork.setStmtId(stmtId);
loadTableWork.setInheritTableSpecs(false);
Task loadPartTask = TaskFactory.get(new MoveWork(
- x.getInputs(), x.getOutputs(), loadTableWork, null, false),
x.getConf(), true);
+ x.getInputs(), x.getOutputs(), loadTableWork, null, false));
copyTask.addDependentTask(loadPartTask);
addPartTask.addDependentTask(loadPartTask);
x.getTasks().add(copyTask);
@@ -831,7 +831,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
if (table.isPartitioned()) {
x.getLOG().debug("table partitioned");
Task ict = createImportCommitTask(
-table.getDbName(), table.getTableName(), writeId, stmtId,
x.getConf(),
+table.getDbName(), table.getTableName(), writeId, stmtId,
AcidUtils.isInsertOnlyTable(table.getParameters()));
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
@@ -868,7 +868,7 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
if (isPartitioned(tblDesc)) {
Task ict = createImportCommitTask(
-tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId,
stmtId, x.getConf(),
+tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
t.addDependentTask(addSinglePartition(fromURI, fs, tblDesc, table,
wh, addPartitionDesc,
@@ -903,11 +903,10 @@ public class ImportSemanticAnalyzer extends
BaseSemanticAnalyzer {
}
private static Task createImportCommitTask(
- String dbName, String tblName, Long writeId, int stmtId, HiveConf conf,
boolean isMmTable) {
+ String dbName, String tblName, Long writeId, int stmtId, boolean
isMmTable) {
// TODO: noop, remove?
-@SuppressWarnings("unchecked")
Task ict =
[2/2] hive git commit: HIVE-18716: Delete unnecessary parameters from TaskFactory (Gergely Hajós via Zoltan Haindrich, Zoltan Haindrich)
HIVE-18716: Delete unnecessary parameters from TaskFactory (Gergely Hajós via
Zoltan Haindrich, Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/94152c99
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/94152c99
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/94152c99
Branch: refs/heads/master
Commit: 94152c9977fa98c294d5062325ee721c09f549e0
Parents: d2d50e6
Author: Gergely Hajós
Authored: Mon Mar 19 10:45:58 2018 +0100
Committer: Zoltan Haindrich
Committed: Mon Mar 19 10:45:58 2018 +0100
--
.../java/org/apache/hadoop/hive/ql/Driver.java | 2 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +-
.../hadoop/hive/ql/exec/ReplCopyTask.java | 4 +-
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 37 ++---
.../ql/exec/repl/bootstrap/ReplLoadTask.java| 4 +-
.../exec/repl/bootstrap/load/LoadDatabase.java | 6 +-
.../exec/repl/bootstrap/load/LoadFunction.java | 2 +-
.../bootstrap/load/table/LoadPartitions.java| 10 +-
.../repl/bootstrap/load/table/LoadTable.java| 4 +-
.../hive/ql/exec/spark/SparkUtilities.java | 4 +-
.../apache/hadoop/hive/ql/hooks/ATSHook.java| 2 +-
.../hive/ql/optimizer/GenMRProcContext.java | 2 +-
.../hive/ql/optimizer/GenMRTableScan1.java | 6 +-
.../hadoop/hive/ql/optimizer/GenMRUnion1.java | 2 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 15 +-
.../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 +-
.../hive/ql/optimizer/StatsOptimizer.java | 2 +-
.../physical/CommonJoinTaskDispatcher.java | 8 +-
.../physical/GenMRSkewJoinProcessor.java| 4 +-
.../physical/GenSparkSkewJoinProcessor.java | 4 +-
.../ql/optimizer/physical/MapJoinResolver.java | 3 +-
.../physical/SortMergeJoinTaskDispatcher.java | 8 +-
.../physical/SparkMapJoinResolver.java | 2 +-
.../spark/SparkSkewJoinProcFactory.java | 2 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 160 +--
.../parse/ExplainSQRewriteSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/FunctionSemanticAnalyzer.java | 6 +-
.../hadoop/hive/ql/parse/GenTezProcContext.java | 4 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 27 ++--
.../hive/ql/parse/LoadSemanticAnalyzer.java | 4 +-
.../hive/ql/parse/MacroSemanticAnalyzer.java| 4 +-
.../hive/ql/parse/ProcessAnalyzeTable.java | 4 +-
.../ql/parse/ReplicationSemanticAnalyzer.java | 18 +--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 12 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 16 +-
.../HiveAuthorizationTaskFactoryImpl.java | 27 ++--
.../repl/load/message/AddForeignKeyHandler.java | 2 +-
.../message/AddNotNullConstraintHandler.java| 2 +-
.../repl/load/message/AddPrimaryKeyHandler.java | 2 +-
.../message/AddUniqueConstraintHandler.java | 2 +-
.../repl/load/message/AlterDatabaseHandler.java | 2 +-
.../load/message/CreateDatabaseHandler.java | 6 +-
.../load/message/CreateFunctionHandler.java | 5 +-
.../load/message/DropConstraintHandler.java | 2 +-
.../repl/load/message/DropDatabaseHandler.java | 2 +-
.../repl/load/message/DropFunctionHandler.java | 2 +-
.../repl/load/message/DropPartitionHandler.java | 3 +-
.../repl/load/message/DropTableHandler.java | 3 +-
.../load/message/RenamePartitionHandler.java| 2 +-
.../repl/load/message/RenameTableHandler.java | 2 +-
.../load/message/TruncatePartitionHandler.java | 3 +-
.../repl/load/message/TruncateTableHandler.java | 3 +-
.../ql/parse/spark/GenSparkProcContext.java | 2 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 4 +-
.../hadoop/hive/ql/plan/ImportTableDesc.java| 4 +-
.../hadoop/hive/ql/exec/TestUtilities.java | 4 +-
.../bootstrap/AddDependencyToLeavesTest.java| 6 +-
60 files changed, 229 insertions(+), 261 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/94152c99/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index eefcaea..75f928b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1854,7 +1854,7 @@ public class Driver implements IDriver {
private void useFetchFromCache(CacheEntry cacheEntry) {
// Change query FetchTask to use new location specified in results cache.
-FetchTask fetc
hive git commit: HIVE-18560: qtests: QTestUtil refactor/split - QOutProcessor (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master d977a9d77 -> db4fe384f
HIVE-18560: qtests: QTestUtil refactor/split - QOutProcessor (Laszlo Bodor via
Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db4fe384
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db4fe384
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db4fe384
Branch: refs/heads/master
Commit: db4fe384fdf57bb89d1f468b68ae6625e6f0ba77
Parents: d977a9d
Author: Laszlo Bodor
Authored: Wed Mar 14 15:32:34 2018 +0100
Committer: Zoltan Haindrich
Committed: Wed Mar 14 15:32:34 2018 +0100
--
.../control/AbstractCoreBlobstoreCliDriver.java | 6 +-
.../apache/hadoop/hive/ql/QOutProcessor.java| 264 +++
.../org/apache/hadoop/hive/ql/QTestUtil.java| 201 +-
.../hadoop/hive/ql/TestQOutProcessor.java | 73 +
.../apache/hadoop/hive/ql/TestQTestUtil.java| 88 ---
.../root_dir_external_table.q.out | 2 +-
.../spark/root_dir_external_table.q.out | 2 +-
7 files changed, 352 insertions(+), 284 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/db4fe384/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index b1d66a5..dd80424 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -27,7 +27,6 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Map;
-import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
@@ -40,6 +39,8 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import com.google.common.base.Strings;
+
public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
protected static QTestUtil qt;
@@ -180,6 +181,7 @@ public abstract class AbstractCoreBlobstoreCliDriver
extends CliAdapter {
+ "-" + String.format("%03d", (int)(Math.random() * 999));
testBlobstorePathUnique = testBlobstorePath + uid;
-qt.addPatternWithMaskComment(testBlobstorePathUnique, String.format("###
%s ###", HCONF_TEST_BLOBSTORE_PATH));
+qt.getQOutProcessor().addPatternWithMaskComment(testBlobstorePathUnique,
+String.format("### %s ###", HCONF_TEST_BLOBSTORE_PATH));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/db4fe384/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
new file mode 100644
index 000..024fa1b
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.ql.QT
hive git commit: HIVE-18944: Groupping sets position is set incorrectly during DPP (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master ed29b4ca9 -> f9e6522c9
HIVE-18944: Groupping sets position is set incorrectly during DPP (Zoltan
Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f9e6522c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f9e6522c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f9e6522c
Branch: refs/heads/master
Commit: f9e6522c9c71afd1543cc1e2533e64033657128b
Parents: ed29b4c
Author: Zoltan Haindrich
Authored: Tue Mar 13 20:00:38 2018 +0100
Committer: Zoltan Haindrich
Committed: Tue Mar 13 20:00:38 2018 +0100
--
.../test/resources/testconfiguration.properties | 1 +
.../DynamicPartitionPruningOptimization.java| 4 +-
.../clientpositive/groupby_groupingset_bug.q| 100 +
.../llap/groupby_groupingset_bug.q.out | 366 +++
4 files changed, 469 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f9e6522c/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index b3e6c3b..99d3817 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -559,6 +559,7 @@ minillaplocal.query.files=\
explainuser_1.q,\
explainuser_4.q,\
groupby2.q,\
+ groupby_groupingset_bug.q,\
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
is_distinct_from.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/f9e6522c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
index 0dbfbe2..9917330 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java
@@ -437,7 +437,7 @@ public class DynamicPartitionPruningOptimization implements
NodeProcessor {
GroupByDesc groupBy =
new GroupByDesc(GroupByDesc.Mode.HASH, outputNames, groupByExprs,
new ArrayList(), false, groupByMemoryUsage,
memoryThreshold,
-null, false, 0, true);
+null, false, -1, true);
GroupByOperator groupByOp = (GroupByOperator)
OperatorFactory.getAndMakeChild(
groupBy, selectOp);
@@ -611,7 +611,7 @@ public class DynamicPartitionPruningOptimization implements
NodeProcessor {
gbOutputNames.add(SemanticAnalyzer.getColumnInternalName(2));
GroupByDesc groupBy = new GroupByDesc(GroupByDesc.Mode.HASH,
gbOutputNames, new ArrayList(), aggs, false,
-groupByMemoryUsage, memoryThreshold, null, false, 0, false);
+groupByMemoryUsage, memoryThreshold, null, false, -1, false);
ArrayList groupbyColInfos = new ArrayList();
groupbyColInfos.add(new ColumnInfo(gbOutputNames.get(0),
key.getTypeInfo(), "", false));
http://git-wip-us.apache.org/repos/asf/hive/blob/f9e6522c/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q
--
diff --git a/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q
b/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q
new file mode 100644
index 000..c6e9a1a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/groupby_groupingset_bug.q
@@ -0,0 +1,100 @@
+set hive.explain.user=true;
+set hive.optimize.index.filter=true;
+set hive.auto.convert.join=true;
+set hive.vectorized.execution.enabled=true;
+
+drop table if exists x1_store_sales;
+drop table if exists x1_date_dim;
+drop table if exists x1_item;
+
+create table x1_store_sales
+(
+ ss_item_sk int
+)
+partitioned by (ss_sold_date_sk int)
+stored as orc;
+
+create table x1_date_dim
+(
+ d_date_sk int,
+ d_month_seq int,
+ d_year int,
+ d_moy int
+)
+stored as orc;
+
+create table x1_item
+(
+ i_item_sk int,
+ i_category char(10),
+ i_current_price decimal(7,2)
+)
+stored as orc;
+
+insert into x1_date_dim values (1,1,2000,2),
+ (1,2,2001,2);
+insert into x1_store_sales partition (ss_sold_date_sk=1) values (1);
+insert into x1_store_sales partition (ss_sold_date_sk=2) values (2);
+
+insert into x1_item values (1,2,1),(1,2,1),(2,2,1);
+
+alter table x1_store_sale
[22/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c22d74c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c22d74c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c22d74c Branch: refs/heads/master Commit: 7c22d74c8d0eb0650adf6e84e0536127c103e46c Parents: 31e36f0 Author: Zoltan Haindrich Authored: Wed Mar 7 08:51:03 2018 +0100 Committer: Zoltan Haindrich Committed: Wed Mar 7 08:51:48 2018 +0100 -- .../hcatalog/mapreduce/HCatOutputFormat.java| 9 - .../listener/DbNotificationListener.java|54 +- .../hcatalog/messaging/AlterIndexMessage.java |35 - .../hcatalog/messaging/CreateIndexMessage.java |35 - .../hcatalog/messaging/DropIndexMessage.java|35 - .../hcatalog/messaging/HCatEventMessage.java|14 +- .../hcatalog/messaging/MessageDeserializer.java |23 - .../hive/hcatalog/messaging/MessageFactory.java |25 - .../messaging/json/JSONAlterIndexMessage.java |89 - .../messaging/json/JSONCreateIndexMessage.java |82 - .../messaging/json/JSONDropIndexMessage.java|82 - .../messaging/json/JSONMessageDeserializer.java |33 - .../messaging/json/JSONMessageFactory.java |48 +- .../listener/DummyRawStoreFailEvent.java|48 - .../listener/TestDbNotificationListener.java| 253 +- .../org/apache/hadoop/hive/ql/QTestUtil.java| 5 - .../upgrade/hive/hive-schema-3.0.0.hive.sql |48 - .../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 - .../apache/hadoop/hive/ql/metadata/Table.java |25 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 4 - .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +- .../AuthorizationPreEventListener.java | 2 - .../hadoop/hive/ql/metadata/TestHive.java | 2 - .../clientpositive/llap/resourceplan.q.out | 104 - .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 9982 ++-- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h| 1318 +-- .../ThriftHiveMetastore_server.skeleton.cpp |30 - .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 4871 .../gen/thrift/gen-cpp/hive_metastore_types.h | 152 - .../hive/metastore/api/AbortTxnsRequest.java|32 +- .../api/AddDefaultConstraintRequest.java|36 +- .../metastore/api/AddDynamicPartitions.java |32 +- .../metastore/api/AddForeignKeyRequest.java |36 +- .../api/AddNotNullConstraintRequest.java|36 +- .../metastore/api/AddPartitionsRequest.java |36 +- .../hive/metastore/api/AddPartitionsResult.java |36 +- .../metastore/api/AddPrimaryKeyRequest.java |36 +- .../api/AddUniqueConstraintRequest.java |36 +- .../hadoop/hive/metastore/api/AggrStats.java|36 +- .../api/AllocateTableWriteIdsRequest.java |32 +- .../api/AllocateTableWriteIdsResponse.java |36 +- .../metastore/api/ClearFileMetadataRequest.java |32 +- .../hive/metastore/api/ClientCapabilities.java |32 +- .../hive/metastore/api/ColumnStatistics.java|36 +- .../hive/metastore/api/CompactionRequest.java |44 +- .../hive/metastore/api/CreationMetadata.java|32 +- .../api/DefaultConstraintsResponse.java |36 +- .../metastore/api/DropPartitionsResult.java |36 +- .../hive/metastore/api/EnvironmentContext.java |44 +- .../hive/metastore/api/FireEventRequest.java|32 +- .../hive/metastore/api/ForeignKeysResponse.java |36 +- .../hadoop/hive/metastore/api/Function.java |36 +- .../metastore/api/GetAllFunctionsResponse.java |36 +- .../api/GetFileMetadataByExprRequest.java |32 +- .../api/GetFileMetadataByExprResult.java|48 +- .../metastore/api/GetFileMetadataRequest.java |32 +- .../metastore/api/GetFileMetadataResult.java|44 +- .../metastore/api/GetOpenTxnsInfoResponse.java |36 +- .../hive/metastore/api/GetOpenTxnsResponse.java |32 +- .../hive/metastore/api/GetTablesRequest.java|32 +- .../hive/metastore/api/GetTablesResult.java |36 +- .../metastore/api/GetValidWriteIdsRequest.java |32 +- .../metastore/api/GetValidWriteIdsResponse.java |36 +- .../api/HeartbeatTxnRangeResponse.java |64 +- .../apache/hadoop/hive/metastore/api/Index.java | 1374 --- .../api/IndexAlreadyExistsException.java| 395 - .../metastore/api/InsertEventRequestData.java |64 +- .../hadoop/hive/metastore/api/LockRequest.java |36 +- .../hive/metastore/api/Materialization.java |32 +- .../api/NotNullConstraintsResponse.java |36 +- .../api/NotificationEventResponse.java |36 +- .../hive/metastore/api/OpenTxnsResponse.java|
[20/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
--
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
index 353d7d9..0f9ba5d 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h
@@ -108,12 +108,6 @@ class ThriftHiveMetastoreIf : virtual public
::facebook::fb303::FacebookService
virtual void partition_name_to_spec(std::map &
_return, const std::string& part_name) = 0;
virtual void markPartitionForEvent(const std::string& db_name, const
std::string& tbl_name, const std::map & part_vals,
const PartitionEventType::type eventType) = 0;
virtual bool isPartitionMarkedForEvent(const std::string& db_name, const
std::string& tbl_name, const std::map & part_vals,
const PartitionEventType::type eventType) = 0;
- virtual void add_index(Index& _return, const Index& new_index, const Table&
index_table) = 0;
- virtual void alter_index(const std::string& dbname, const std::string&
base_tbl_name, const std::string& idx_name, const Index& new_idx) = 0;
- virtual bool drop_index_by_name(const std::string& db_name, const
std::string& tbl_name, const std::string& index_name, const bool deleteData) =
0;
- virtual void get_index_by_name(Index& _return, const std::string& db_name,
const std::string& tbl_name, const std::string& index_name) = 0;
- virtual void get_indexes(std::vector & _return, const std::string&
db_name, const std::string& tbl_name, const int16_t max_indexes) = 0;
- virtual void get_index_names(std::vector & _return, const
std::string& db_name, const std::string& tbl_name, const int16_t max_indexes) =
0;
virtual void get_primary_keys(PrimaryKeysResponse& _return, const
PrimaryKeysRequest& request) = 0;
virtual void get_foreign_keys(ForeignKeysResponse& _return, const
ForeignKeysRequest& request) = 0;
virtual void get_unique_constraints(UniqueConstraintsResponse& _return,
const UniqueConstraintsRequest& request) = 0;
@@ -506,25 +500,6 @@ class ThriftHiveMetastoreNull : virtual public
ThriftHiveMetastoreIf , virtual p
bool _return = false;
return _return;
}
- void add_index(Index& /* _return */, const Index& /* new_index */, const
Table& /* index_table */) {
-return;
- }
- void alter_index(const std::string& /* dbname */, const std::string& /*
base_tbl_name */, const std::string& /* idx_name */, const Index& /* new_idx
*/) {
-return;
- }
- bool drop_index_by_name(const std::string& /* db_name */, const std::string&
/* tbl_name */, const std::string& /* index_name */, const bool /* deleteData
*/) {
-bool _return = false;
-return _return;
- }
- void get_index_by_name(Index& /* _return */, const std::string& /* db_name
*/, const std::string& /* tbl_name */, const std::string& /* index_name */) {
-return;
- }
- void get_indexes(std::vector & /* _return */, const std::string& /*
db_name */, const std::string& /* tbl_name */, const int16_t /* max_indexes */)
{
-return;
- }
- void get_index_names(std::vector & /* _return */, const
std::string& /* db_name */, const std::string& /* tbl_name */, const int16_t /*
max_indexes */) {
-return;
- }
void get_primary_keys(PrimaryKeysResponse& /* _return */, const
PrimaryKeysRequest& /* request */) {
return;
}
@@ -11644,993 +11619,118 @@ class
ThriftHiveMetastore_partition_name_to_spec_pargs {
};
-typedef struct _ThriftHiveMetastore_partition_name_to_spec_result__isset {
- _ThriftHiveMetastore_partition_name_to_spec_result__isset() :
success(false), o1(false) {}
- bool success :1;
- bool o1 :1;
-} _ThriftHiveMetastore_partition_name_to_spec_result__isset;
-
-class ThriftHiveMetastore_partition_name_to_spec_result {
- public:
-
- ThriftHiveMetastore_partition_name_to_spec_result(const
ThriftHiveMetastore_partition_name_to_spec_result&);
- ThriftHiveMetastore_partition_name_to_spec_result& operator=(const
ThriftHiveMetastore_partition_name_to_spec_result&);
- ThriftHiveMetastore_partition_name_to_spec_result() {
- }
-
- virtual ~ThriftHiveMetastore_partition_name_to_spec_result() throw();
- std::map success;
- MetaException o1;
-
- _ThriftHiveMetastore_partition_name_to_spec_result__isset __isset;
-
- void __set_success(const std::map & val);
-
- void __set_o1(const MetaException& val);
-
- bool operator == (const ThriftHiveMetastore_partition_name_to_spec_result &
rhs) const
- {
-if (!(success == rhs.success))
- return false;
-if (!(o1 == rhs.o1))
- return false;
-return true;
- }
- bool operator != (const ThriftHiveMetastore_partition_name_to_spec_result
&rhs) const {
-return !(*this == rhs);
- }
-
- bool operator < (const ThriftHiveMetastore_partition_name_to_spec_result & )
con
[08/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 883ecfe..b24c5f9 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -4493,199 +4493,6 @@ class PartitionSpec:
def __ne__(self, other):
return not (self == other)
-class Index:
- """
- Attributes:
- - indexName
- - indexHandlerClass
- - dbName
- - origTableName
- - createTime
- - lastAccessTime
- - indexTableName
- - sd
- - parameters
- - deferredRebuild
- """
-
- thrift_spec = (
-None, # 0
-(1, TType.STRING, 'indexName', None, None, ), # 1
-(2, TType.STRING, 'indexHandlerClass', None, None, ), # 2
-(3, TType.STRING, 'dbName', None, None, ), # 3
-(4, TType.STRING, 'origTableName', None, None, ), # 4
-(5, TType.I32, 'createTime', None, None, ), # 5
-(6, TType.I32, 'lastAccessTime', None, None, ), # 6
-(7, TType.STRING, 'indexTableName', None, None, ), # 7
-(8, TType.STRUCT, 'sd', (StorageDescriptor,
StorageDescriptor.thrift_spec), None, ), # 8
-(9, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None,
), # 9
-(10, TType.BOOL, 'deferredRebuild', None, None, ), # 10
- )
-
- def __init__(self, indexName=None, indexHandlerClass=None, dbName=None,
origTableName=None, createTime=None, lastAccessTime=None, indexTableName=None,
sd=None, parameters=None, deferredRebuild=None,):
-self.indexName = indexName
-self.indexHandlerClass = indexHandlerClass
-self.dbName = dbName
-self.origTableName = origTableName
-self.createTime = createTime
-self.lastAccessTime = lastAccessTime
-self.indexTableName = indexTableName
-self.sd = sd
-self.parameters = parameters
-self.deferredRebuild = deferredRebuild
-
- def read(self, iprot):
-if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is
not None and fastbinary is not None:
- fastbinary.decode_binary(self, iprot.trans, (self.__class__,
self.thrift_spec))
- return
-iprot.readStructBegin()
-while True:
- (fname, ftype, fid) = iprot.readFieldBegin()
- if ftype == TType.STOP:
-break
- if fid == 1:
-if ftype == TType.STRING:
- self.indexName = iprot.readString()
-else:
- iprot.skip(ftype)
- elif fid == 2:
-if ftype == TType.STRING:
- self.indexHandlerClass = iprot.readString()
-else:
- iprot.skip(ftype)
- elif fid == 3:
-if ftype == TType.STRING:
- self.dbName = iprot.readString()
-else:
- iprot.skip(ftype)
- elif fid == 4:
-if ftype == TType.STRING:
- self.origTableName = iprot.readString()
-else:
- iprot.skip(ftype)
- elif fid == 5:
-if ftype == TType.I32:
- self.createTime = iprot.readI32()
-else:
- iprot.skip(ftype)
- elif fid == 6:
-if ftype == TType.I32:
- self.lastAccessTime = iprot.readI32()
-else:
- iprot.skip(ftype)
- elif fid == 7:
-if ftype == TType.STRING:
- self.indexTableName = iprot.readString()
-else:
- iprot.skip(ftype)
- elif fid == 8:
-if ftype == TType.STRUCT:
- self.sd = StorageDescriptor()
- self.sd.read(iprot)
-else:
- iprot.skip(ftype)
- elif fid == 9:
-if ftype == TType.MAP:
- self.parameters = {}
- (_ktype231, _vtype232, _size230 ) = iprot.readMapBegin()
- for _i234 in xrange(_size230):
-_key235 = iprot.readString()
-_val236 = iprot.readString()
-self.parameters[_key235] = _val236
- iprot.readMapEnd()
-else:
- iprot.skip(ftype)
- elif fid == 10:
-if ftype == TType.BOOL:
- self.deferredRebuild = iprot.readBool()
-else:
- iprot.skip(ftype)
- else:
-iprot.skip(ftype)
- iprot.readFieldEnd()
-iprot.readStructEnd()
-
- def write(self, oprot):
-if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
self.thrift_spec is not None and fastbinary is not None:
- oprot.trans.write(fastbinary.encode_binary(self, (self.__class__,
self.thrift_spec)))
- return
-oprot.writeStructBegin('Index')
-if self.indexName is not None:
- oprot.writeFieldBegin('indexName', TType.STRING, 1)
- oprot.writeString(self.indexName)
- oprot.writeFieldEnd()
-if self.indexHandlerClass is not None:
- oprot.writeFieldBegin('indexHandlerClas
[07/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
--
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 8438728..29ad1bc 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -982,40 +982,6 @@ class PartitionSpec
::Thrift::Struct.generate_accessors self
end
-class Index
- include ::Thrift::Struct, ::Thrift::Struct_Union
- INDEXNAME = 1
- INDEXHANDLERCLASS = 2
- DBNAME = 3
- ORIGTABLENAME = 4
- CREATETIME = 5
- LASTACCESSTIME = 6
- INDEXTABLENAME = 7
- SD = 8
- PARAMETERS = 9
- DEFERREDREBUILD = 10
-
- FIELDS = {
-INDEXNAME => {:type => ::Thrift::Types::STRING, :name => 'indexName'},
-INDEXHANDLERCLASS => {:type => ::Thrift::Types::STRING, :name =>
'indexHandlerClass'},
-DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
-ORIGTABLENAME => {:type => ::Thrift::Types::STRING, :name =>
'origTableName'},
-CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
-LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name =>
'lastAccessTime'},
-INDEXTABLENAME => {:type => ::Thrift::Types::STRING, :name =>
'indexTableName'},
-SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class =>
::StorageDescriptor},
-PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key
=> {:type => ::Thrift::Types::STRING}, :value => {:type =>
::Thrift::Types::STRING}},
-DEFERREDREBUILD => {:type => ::Thrift::Types::BOOL, :name =>
'deferredRebuild'}
- }
-
- def struct_fields; FIELDS; end
-
- def validate
- end
-
- ::Thrift::Struct.generate_accessors self
-end
-
class BooleanColumnStatsData
include ::Thrift::Struct, ::Thrift::Struct_Union
NUMTRUES = 1
@@ -4482,27 +4448,6 @@ class NoSuchObjectException < ::Thrift::Exception
::Thrift::Struct.generate_accessors self
end
-class IndexAlreadyExistsException < ::Thrift::Exception
- include ::Thrift::Struct, ::Thrift::Struct_Union
- def initialize(message=nil)
-super()
-self.message = message
- end
-
- MESSAGE = 1
-
- FIELDS = {
-MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'}
- }
-
- def struct_fields; FIELDS; end
-
- def validate
- end
-
- ::Thrift::Struct.generate_accessors self
-end
-
class InvalidOperationException < ::Thrift::Exception
include ::Thrift::Struct, ::Thrift::Struct_Union
def initialize(message=nil)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
index b0e84d9..4990fed 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb
@@ -1469,107 +1469,6 @@ module ThriftHiveMetastore
raise
::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT,
'isPartitionMarkedForEvent failed: unknown result')
end
-def add_index(new_index, index_table)
- send_add_index(new_index, index_table)
- return recv_add_index()
-end
-
-def send_add_index(new_index, index_table)
- send_message('add_index', Add_index_args, :new_index => new_index,
:index_table => index_table)
-end
-
-def recv_add_index()
- result = receive_message(Add_index_result)
- return result.success unless result.success.nil?
- raise result.o1 unless result.o1.nil?
- raise result.o2 unless result.o2.nil?
- raise result.o3 unless result.o3.nil?
- raise
::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT,
'add_index failed: unknown result')
-end
-
-def alter_index(dbname, base_tbl_name, idx_name, new_idx)
- send_alter_index(dbname, base_tbl_name, idx_name, new_idx)
- recv_alter_index()
-end
-
-def send_alter_index(dbname, base_tbl_name, idx_name, new_idx)
- send_message('alter_index', Alter_index_args, :dbname => dbname,
:base_tbl_name => base_tbl_name, :idx_name => idx_name, :new_idx => new_idx)
-end
-
-def recv_alter_index()
- result = receive_message(Alter_index_result)
- raise result.o1 unless result.o1.nil?
- raise result.o2 unless result.o2.nil?
- return
-end
-
-def drop_index_by_name(db_name, tbl_name, index_name, deleteData)
- send_drop_index_by_name(db_name, tbl_name, index_name, deleteData)
- return recv_drop_index_by_name()
-end
-
-def send_drop_index_by_name(db_name, tbl_name, index_name, deleteDat
[12/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
index 57b6bfa..02242a7 100644
---
a/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
+++
b/standalone-metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php
@@ -753,60 +753,6 @@ interface ThriftHiveMetastoreIf extends \FacebookServiceIf
{
*/
public function isPartitionMarkedForEvent($db_name, $tbl_name, array
$part_vals, $eventType);
/**
- * @param \metastore\Index $new_index
- * @param \metastore\Table $index_table
- * @return \metastore\Index
- * @throws \metastore\InvalidObjectException
- * @throws \metastore\AlreadyExistsException
- * @throws \metastore\MetaException
- */
- public function add_index(\metastore\Index $new_index, \metastore\Table
$index_table);
- /**
- * @param string $dbname
- * @param string $base_tbl_name
- * @param string $idx_name
- * @param \metastore\Index $new_idx
- * @throws \metastore\InvalidOperationException
- * @throws \metastore\MetaException
- */
- public function alter_index($dbname, $base_tbl_name, $idx_name,
\metastore\Index $new_idx);
- /**
- * @param string $db_name
- * @param string $tbl_name
- * @param string $index_name
- * @param bool $deleteData
- * @return bool
- * @throws \metastore\NoSuchObjectException
- * @throws \metastore\MetaException
- */
- public function drop_index_by_name($db_name, $tbl_name, $index_name,
$deleteData);
- /**
- * @param string $db_name
- * @param string $tbl_name
- * @param string $index_name
- * @return \metastore\Index
- * @throws \metastore\MetaException
- * @throws \metastore\NoSuchObjectException
- */
- public function get_index_by_name($db_name, $tbl_name, $index_name);
- /**
- * @param string $db_name
- * @param string $tbl_name
- * @param int $max_indexes
- * @return \metastore\Index[]
- * @throws \metastore\NoSuchObjectException
- * @throws \metastore\MetaException
- */
- public function get_indexes($db_name, $tbl_name, $max_indexes);
- /**
- * @param string $db_name
- * @param string $tbl_name
- * @param int $max_indexes
- * @return string[]
- * @throws \metastore\MetaException
- */
- public function get_index_names($db_name, $tbl_name, $max_indexes);
- /**
* @param \metastore\PrimaryKeysRequest $request
* @return \metastore\PrimaryKeysResponse
* @throws \metastore\MetaException
@@ -6463,358 +6409,6 @@ class ThriftHiveMetastoreClient extends
\FacebookServiceClient implements \metas
throw new \Exception("isPartitionMarkedForEvent failed: unknown result");
}
- public function add_index(\metastore\Index $new_index, \metastore\Table
$index_table)
- {
-$this->send_add_index($new_index, $index_table);
-return $this->recv_add_index();
- }
-
- public function send_add_index(\metastore\Index $new_index, \metastore\Table
$index_table)
- {
-$args = new \metastore\ThriftHiveMetastore_add_index_args();
-$args->new_index = $new_index;
-$args->index_table = $index_table;
-$bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_write_binary');
-if ($bin_accel)
-{
- thrift_protocol_write_binary($this->output_, 'add_index',
TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite());
-}
-else
-{
- $this->output_->writeMessageBegin('add_index', TMessageType::CALL,
$this->seqid_);
- $args->write($this->output_);
- $this->output_->writeMessageEnd();
- $this->output_->getTransport()->flush();
-}
- }
-
- public function recv_add_index()
- {
-$bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) &&
function_exists('thrift_protocol_read_binary');
-if ($bin_accel) $result = thrift_protocol_read_binary($this->input_,
'\metastore\ThriftHiveMetastore_add_index_result',
$this->input_->isStrictRead());
-else
-{
- $rseqid = 0;
- $fname = null;
- $mtype = 0;
-
- $this->input_->readMessageBegin($fname, $mtype, $rseqid);
- if ($mtype == TMessageType::EXCEPTION) {
-$x = new TApplicationException();
-$x->read($this->input_);
-$this->input_->readMessageEnd();
-throw $x;
- }
- $result = new \metastore\ThriftHiveMetastore_add_index_result();
- $result->read($this->input_);
- $this->input_->readMessageEnd();
-}
-if ($result->success !== null) {
- return $result->success;
-}
-if ($result->o1 !== null) {
- throw $result->o1;
-}
-if ($result->o2 !== null) {
- throw $result->o2;
-}
-if ($result->o3 !== null) {
- throw $
[04/22] hive git commit: HIVE-17626: Query reoptimization using cached runtime statistics (Zoltan Haindrich reviewed by Ashutosh Chauhan)
HIVE-17626: Query reoptimization using cached runtime statistics (Zoltan Haindrich reviewed by Ashutosh Chauhan) Signed-off-by: Zoltan Haindrich Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31e36f01 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31e36f01 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31e36f01 Branch: refs/heads/master Commit: 31e36f01952e7bcc803060351758798f9af9e0b3 Parents: cd8eda8 Author: Zoltan Haindrich Authored: Wed Mar 7 08:44:14 2018 +0100 Committer: Zoltan Haindrich Committed: Wed Mar 7 08:44:14 2018 +0100 -- .../org/apache/hadoop/hive/conf/HiveConf.java | 35 +- data/conf/llap/hive-site.xml| 5 + .../druid/serde/DruidScanQueryRecordReader.java | 2 +- .../test/resources/testconfiguration.properties | 3 + .../org/apache/hadoop/hive/ql/QTestUtil.java| 2 +- .../apache/hadoop/hive/ql/TestQTestUtil.java| 2 +- .../java/org/apache/hadoop/hive/ql/Context.java | 46 ++- .../java/org/apache/hadoop/hive/ql/Driver.java | 55 ++- .../apache/hadoop/hive/ql/DriverFactory.java| 51 ++- .../org/apache/hadoop/hive/ql/HookRunner.java | 5 + .../java/org/apache/hadoop/hive/ql/IDriver.java | 7 + .../hive/ql/cache/results/CacheUsage.java | 2 +- .../ql/cache/results/QueryResultsCache.java | 2 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 1 + .../hive/ql/exec/MaterializedViewDesc.java | 2 +- .../hive/ql/exec/MaterializedViewTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Operator.java| 54 ++- .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 21 +- .../hive/ql/exec/tez/HiveInputCounters.java | 2 +- .../hive/ql/exec/tez/LlapObjectSubCache.java| 2 +- .../VectorReduceSinkCommonOperator.java | 15 +- .../VectorReduceSinkEmptyKeyOperator.java | 38 -- .../VectorReduceSinkObjectHashOperator.java | 28 -- .../VectorReduceSinkUniformHashOperator.java| 35 -- .../hive/ql/hooks/PrivateHookContext.java | 50 +++ .../hadoop/hive/ql/metadata/HiveException.java | 1 + .../hive/ql/optimizer/SharedWorkOptimizer.java | 3 + .../HiveRelOpMaterializationValidator.java | 2 +- .../hive/ql/optimizer/physical/Vectorizer.java | 32 +- .../ql/optimizer/signature/OpSignature.java | 94 + .../ql/optimizer/signature/OpTreeSignature.java | 90 + .../signature/OpTreeSignatureFactory.java | 67 .../hive/ql/optimizer/signature/Signature.java | 36 ++ .../ql/optimizer/signature/SignatureUtils.java | 95 + .../spark/SparkPartitionPruningSinkDesc.java| 3 + .../stats/annotation/StatsRulesProcFactory.java | 61 ++- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 1 + .../apache/hadoop/hive/ql/parse/HiveParser.g| 13 +- .../hadoop/hive/ql/parse/IdentifiersParser.g| 1 + .../hive/ql/plan/AbstractOperatorDesc.java | 12 +- .../hadoop/hive/ql/plan/AppMasterEventDesc.java | 7 +- .../hive/ql/plan/CommonMergeJoinDesc.java | 3 + .../hive/ql/plan/DynamicPruningEventDesc.java | 5 + .../hadoop/hive/ql/plan/FileSinkDesc.java | 18 + .../apache/hadoop/hive/ql/plan/FilterDesc.java | 5 + .../apache/hadoop/hive/ql/plan/GroupByDesc.java | 13 +- .../hadoop/hive/ql/plan/HashTableSinkDesc.java | 5 + .../hadoop/hive/ql/plan/JoinCondDesc.java | 6 +- .../apache/hadoop/hive/ql/plan/JoinDesc.java| 8 + .../hive/ql/plan/LateralViewJoinDesc.java | 3 + .../apache/hadoop/hive/ql/plan/LimitDesc.java | 3 + .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 6 + .../hadoop/hive/ql/plan/OperatorDesc.java | 3 + .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 13 +- .../apache/hadoop/hive/ql/plan/ScriptDesc.java | 8 +- .../apache/hadoop/hive/ql/plan/SelectDesc.java | 5 + .../apache/hadoop/hive/ql/plan/Statistics.java | 27 +- .../hadoop/hive/ql/plan/TableScanDesc.java | 19 + .../apache/hadoop/hive/ql/plan/UDTFDesc.java| 7 +- .../hive/ql/plan/mapper/EmptyStatsSource.java | 28 ++ .../hive/ql/plan/mapper/GroupTransformer.java | 25 ++ .../hadoop/hive/ql/plan/mapper/PlanMapper.java | 112 ++ .../hive/ql/plan/mapper/PlanMapperProcess.java | 47 +++ .../hive/ql/plan/mapper/RuntimeStatsSource.java | 29 ++ .../plan/mapper/SimpleRuntimeStatsSource.java | 65 .../hadoop/hive/ql/plan/mapper/StatsSource.java | 25 ++ .../hive/ql/reexec/IReExecutionPlugin.java | 64 .../hadoop/hive/ql/reexec/ReExecDriver.java | 263 + .../ql/reexec/ReExecutionOverlayPlugin.java | 83 .../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 138 +++ .../hadoop/hive/ql/stats/OperatorStats.java | 43 +++ .../hive/ql/stats/OperatorStatsReaderHook.java | 96 + .../ql/udf/generic/GenericUDFAssertTrueOOM.java | 74 .../GenericUDFEnforceNotNullCon
[17/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ForeignKeysResponse.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ForeignKeysResponse.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ForeignKeysResponse.java
index 75b2404..081adeb 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ForeignKeysResponse.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ForeignKeysResponse.java
@@ -354,14 +354,14 @@ import org.slf4j.LoggerFactory;
case 1: // FOREIGN_KEYS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
-org.apache.thrift.protocol.TList _list330 =
iprot.readListBegin();
-struct.foreignKeys = new
ArrayList(_list330.size);
-SQLForeignKey _elem331;
-for (int _i332 = 0; _i332 < _list330.size; ++_i332)
+org.apache.thrift.protocol.TList _list320 =
iprot.readListBegin();
+struct.foreignKeys = new
ArrayList(_list320.size);
+SQLForeignKey _elem321;
+for (int _i322 = 0; _i322 < _list320.size; ++_i322)
{
- _elem331 = new SQLForeignKey();
- _elem331.read(iprot);
- struct.foreignKeys.add(_elem331);
+ _elem321 = new SQLForeignKey();
+ _elem321.read(iprot);
+ struct.foreignKeys.add(_elem321);
}
iprot.readListEnd();
}
@@ -387,9 +387,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(FOREIGN_KEYS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.foreignKeys.size()));
- for (SQLForeignKey _iter333 : struct.foreignKeys)
+ for (SQLForeignKey _iter323 : struct.foreignKeys)
{
-_iter333.write(oprot);
+_iter323.write(oprot);
}
oprot.writeListEnd();
}
@@ -414,9 +414,9 @@ import org.slf4j.LoggerFactory;
TTupleProtocol oprot = (TTupleProtocol) prot;
{
oprot.writeI32(struct.foreignKeys.size());
-for (SQLForeignKey _iter334 : struct.foreignKeys)
+for (SQLForeignKey _iter324 : struct.foreignKeys)
{
- _iter334.write(oprot);
+ _iter324.write(oprot);
}
}
}
@@ -425,14 +425,14 @@ import org.slf4j.LoggerFactory;
public void read(org.apache.thrift.protocol.TProtocol prot,
ForeignKeysResponse struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
{
-org.apache.thrift.protocol.TList _list335 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
iprot.readI32());
-struct.foreignKeys = new ArrayList(_list335.size);
-SQLForeignKey _elem336;
-for (int _i337 = 0; _i337 < _list335.size; ++_i337)
+org.apache.thrift.protocol.TList _list325 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
iprot.readI32());
+struct.foreignKeys = new ArrayList(_list325.size);
+SQLForeignKey _elem326;
+for (int _i327 = 0; _i327 < _list325.size; ++_i327)
{
- _elem336 = new SQLForeignKey();
- _elem336.read(iprot);
- struct.foreignKeys.add(_elem336);
+ _elem326 = new SQLForeignKey();
+ _elem326.read(iprot);
+ struct.foreignKeys.add(_elem326);
}
}
struct.setForeignKeysIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
index 32e94cc..5e785df 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Function.java
@@ -997,14 +997,14 @@ import org.slf4j.LoggerFactory;
case 8: // RESOURCE_URIS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
-org.apache.thrift.protocol.TList _list532 =
iprot.readListBegin();
-struct.resourceUris = new
ArrayList(_list532.size);
-ResourceUri _elem533;
[21/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
index 1510d4f..32a7086 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp
@@ -1240,14 +1240,14 @@ uint32_t
ThriftHiveMetastore_get_databases_result::read(::apache::thrift::protoc
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size;
-::apache::thrift::protocol::TType _etype1114;
-xfer += iprot->readListBegin(_etype1114, _size);
-this->success.resize(_size);
-uint32_t _i1115;
-for (_i1115 = 0; _i1115 < _size; ++_i1115)
+uint32_t _size1099;
+::apache::thrift::protocol::TType _etype1102;
+xfer += iprot->readListBegin(_etype1102, _size1099);
+this->success.resize(_size1099);
+uint32_t _i1103;
+for (_i1103 = 0; _i1103 < _size1099; ++_i1103)
{
- xfer += iprot->readString(this->success[_i1115]);
+ xfer += iprot->readString(this->success[_i1103]);
}
xfer += iprot->readListEnd();
}
@@ -1286,10 +1286,10 @@ uint32_t
ThriftHiveMetastore_get_databases_result::write(::apache::thrift::proto
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1116;
- for (_iter1116 = this->success.begin(); _iter1116 !=
this->success.end(); ++_iter1116)
+ std::vector ::const_iterator _iter1104;
+ for (_iter1104 = this->success.begin(); _iter1104 !=
this->success.end(); ++_iter1104)
{
-xfer += oprot->writeString((*_iter1116));
+xfer += oprot->writeString((*_iter1104));
}
xfer += oprot->writeListEnd();
}
@@ -1334,14 +1334,14 @@ uint32_t
ThriftHiveMetastore_get_databases_presult::read(::apache::thrift::proto
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
(*(this->success)).clear();
-uint32_t _size1117;
-::apache::thrift::protocol::TType _etype1120;
-xfer += iprot->readListBegin(_etype1120, _size1117);
-(*(this->success)).resize(_size1117);
-uint32_t _i1121;
-for (_i1121 = 0; _i1121 < _size1117; ++_i1121)
+uint32_t _size1105;
+::apache::thrift::protocol::TType _etype1108;
+xfer += iprot->readListBegin(_etype1108, _size1105);
+(*(this->success)).resize(_size1105);
+uint32_t _i1109;
+for (_i1109 = 0; _i1109 < _size1105; ++_i1109)
{
- xfer += iprot->readString((*(this->success))[_i1121]);
+ xfer += iprot->readString((*(this->success))[_i1109]);
}
xfer += iprot->readListEnd();
}
@@ -1458,14 +1458,14 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::read(::apache::thrift::pr
if (ftype == ::apache::thrift::protocol::T_LIST) {
{
this->success.clear();
-uint32_t _size1122;
-::apache::thrift::protocol::TType _etype1125;
-xfer += iprot->readListBegin(_etype1125, _size1122);
-this->success.resize(_size1122);
-uint32_t _i1126;
-for (_i1126 = 0; _i1126 < _size1122; ++_i1126)
+uint32_t _size1110;
+::apache::thrift::protocol::TType _etype1113;
+xfer += iprot->readListBegin(_etype1113, _size1110);
+this->success.resize(_size1110);
+uint32_t _i1114;
+for (_i1114 = 0; _i1114 < _size1110; ++_i1114)
{
- xfer += iprot->readString(this->success[_i1126]);
+ xfer += iprot->readString(this->success[_i1114]);
}
xfer += iprot->readListEnd();
}
@@ -1504,10 +1504,10 @@ uint32_t
ThriftHiveMetastore_get_all_databases_result::write(::apache::thrift::p
xfer += oprot->writeFieldBegin("success",
::apache::thrift::protocol::T_LIST, 0);
{
xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING,
static_cast(this->success.size()));
- std::vector ::const_iterator _iter1127;
- for (_iter1127 = this->success.begin(); _iter1127 !=
this->success.end(); ++_iter1127)
+ std::vector ::const_iterator _iter1115;
+ for (_iter1115 = this->success.begin(); _iter1115 !=
this->success.end(); ++_iter1115)
{
-xfer += opro
[05/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerWithOldConf.java -- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerWithOldConf.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerWithOldConf.java index a9d9e78..508b2c6 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerWithOldConf.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerWithOldConf.java @@ -18,69 +18,19 @@ package org.apache.hadoop.hive.metastore; -import com.google.common.collect.Lists; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PartitionEventType; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; -import org.apache.hadoop.hive.metastore.client.builder.IndexBuilder; -import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; -import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.apache.hadoop.hive.metastore.events.AddIndexEvent; -import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; -import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; -import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; -import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; -import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.CreateTableEvent; -import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.DropIndexEvent; -import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; -import org.apache.hadoop.hive.metastore.events.DropTableEvent; -import org.apache.hadoop.hive.metastore.events.ListenerEvent; -import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; -import org.apache.hadoop.hive.metastore.events.PreAddIndexEvent; -import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; -import org.apache.hadoop.hive.metastore.events.PreAlterIndexEvent; -import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; -import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; -import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent; -import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.PreDropIndexEvent; -import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent; -import org.apache.hadoop.hive.metastore.events.PreDropTableEvent; -import org.apache.hadoop.hive.metastore.events.PreEventContext; -import org.apache.hadoop.hive.metastore.events.PreLoadPartitionDoneEvent; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - /** * Mostly same tests as TestMetaStoreEventListener, but using old hive conf values instead of new * metastore conf values. http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java -- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index f9bd495..b7f7be92 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/src/test/java/org/apache
[15/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionValuesRequest.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionValuesRequest.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionValuesRequest.java
index 93a5034..bb9bc51 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionValuesRequest.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/PartitionValuesRequest.java
@@ -961,14 +961,14 @@ import org.slf4j.LoggerFactory;
case 3: // PARTITION_KEYS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
-org.apache.thrift.protocol.TList _list500 =
iprot.readListBegin();
-struct.partitionKeys = new
ArrayList(_list500.size);
-FieldSchema _elem501;
-for (int _i502 = 0; _i502 < _list500.size; ++_i502)
+org.apache.thrift.protocol.TList _list490 =
iprot.readListBegin();
+struct.partitionKeys = new
ArrayList(_list490.size);
+FieldSchema _elem491;
+for (int _i492 = 0; _i492 < _list490.size; ++_i492)
{
- _elem501 = new FieldSchema();
- _elem501.read(iprot);
- struct.partitionKeys.add(_elem501);
+ _elem491 = new FieldSchema();
+ _elem491.read(iprot);
+ struct.partitionKeys.add(_elem491);
}
iprot.readListEnd();
}
@@ -996,14 +996,14 @@ import org.slf4j.LoggerFactory;
case 6: // PARTITION_ORDER
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
-org.apache.thrift.protocol.TList _list503 =
iprot.readListBegin();
-struct.partitionOrder = new
ArrayList(_list503.size);
-FieldSchema _elem504;
-for (int _i505 = 0; _i505 < _list503.size; ++_i505)
+org.apache.thrift.protocol.TList _list493 =
iprot.readListBegin();
+struct.partitionOrder = new
ArrayList(_list493.size);
+FieldSchema _elem494;
+for (int _i495 = 0; _i495 < _list493.size; ++_i495)
{
- _elem504 = new FieldSchema();
- _elem504.read(iprot);
- struct.partitionOrder.add(_elem504);
+ _elem494 = new FieldSchema();
+ _elem494.read(iprot);
+ struct.partitionOrder.add(_elem494);
}
iprot.readListEnd();
}
@@ -1055,9 +1055,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(PARTITION_KEYS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.partitionKeys.size()));
- for (FieldSchema _iter506 : struct.partitionKeys)
+ for (FieldSchema _iter496 : struct.partitionKeys)
{
-_iter506.write(oprot);
+_iter496.write(oprot);
}
oprot.writeListEnd();
}
@@ -1080,9 +1080,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(PARTITION_ORDER_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.partitionOrder.size()));
-for (FieldSchema _iter507 : struct.partitionOrder)
+for (FieldSchema _iter497 : struct.partitionOrder)
{
- _iter507.write(oprot);
+ _iter497.write(oprot);
}
oprot.writeListEnd();
}
@@ -1120,9 +1120,9 @@ import org.slf4j.LoggerFactory;
oprot.writeString(struct.tblName);
{
oprot.writeI32(struct.partitionKeys.size());
-for (FieldSchema _iter508 : struct.partitionKeys)
+for (FieldSchema _iter498 : struct.partitionKeys)
{
- _iter508.write(oprot);
+ _iter498.write(oprot);
}
}
BitSet optionals = new BitSet();
@@ -1151,9 +1151,9 @@ import org.slf4j.LoggerFactory;
if (struct.isSetPartitionOrder()) {
{
oprot.writeI32(struct.partitionOrder.size());
- for (FieldSchema _iter509 : struct.partitionOrder)
+ for (FieldSchema _iter499 : struct.partitionOrder)
{
-_iter509.write(oprot);
+_iter499.write(oprot);
}
}
}
@@ -1173,14 +1173,14 @@ import org.slf4j.LoggerFactory;
struct.tblName = iprot.readString();
struct.setTblNameIsSe
[16/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java
deleted file mode 100644
index 8f2af25..000
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Index.java
+++ /dev/null
@@ -1,1374 +0,0 @@
-/**
- * Autogenerated by Thrift Compiler (0.9.3)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.hadoop.hive.metastore.api;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import org.apache.thrift.async.AsyncMethodCallback;
-import org.apache.thrift.server.AbstractNonblockingServer.*;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import javax.annotation.Generated;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
-@org.apache.hadoop.classification.InterfaceAudience.Public
@org.apache.hadoop.classification.InterfaceStability.Stable public class Index
implements org.apache.thrift.TBase, java.io.Serializable,
Cloneable, Comparable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("Index");
-
- private static final org.apache.thrift.protocol.TField INDEX_NAME_FIELD_DESC
= new org.apache.thrift.protocol.TField("indexName",
org.apache.thrift.protocol.TType.STRING, (short)1);
- private static final org.apache.thrift.protocol.TField
INDEX_HANDLER_CLASS_FIELD_DESC = new
org.apache.thrift.protocol.TField("indexHandlerClass",
org.apache.thrift.protocol.TType.STRING, (short)2);
- private static final org.apache.thrift.protocol.TField DB_NAME_FIELD_DESC =
new org.apache.thrift.protocol.TField("dbName",
org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField
ORIG_TABLE_NAME_FIELD_DESC = new
org.apache.thrift.protocol.TField("origTableName",
org.apache.thrift.protocol.TType.STRING, (short)4);
- private static final org.apache.thrift.protocol.TField
CREATE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("createTime",
org.apache.thrift.protocol.TType.I32, (short)5);
- private static final org.apache.thrift.protocol.TField
LAST_ACCESS_TIME_FIELD_DESC = new
org.apache.thrift.protocol.TField("lastAccessTime",
org.apache.thrift.protocol.TType.I32, (short)6);
- private static final org.apache.thrift.protocol.TField
INDEX_TABLE_NAME_FIELD_DESC = new
org.apache.thrift.protocol.TField("indexTableName",
org.apache.thrift.protocol.TType.STRING, (short)7);
- private static final org.apache.thrift.protocol.TField SD_FIELD_DESC = new
org.apache.thrift.protocol.TField("sd",
org.apache.thrift.protocol.TType.STRUCT, (short)8);
- private static final org.apache.thrift.protocol.TField PARAMETERS_FIELD_DESC
= new org.apache.thrift.protocol.TField("parameters",
org.apache.thrift.protocol.TType.MAP, (short)9);
- private static final org.apache.thrift.protocol.TField
DEFERRED_REBUILD_FIELD_DESC = new
org.apache.thrift.protocol.TField("deferredRebuild",
org.apache.thrift.protocol.TType.BOOL, (short)10);
-
- private static final Map, SchemeFactory> schemes =
new HashMap, SchemeFactory>();
- static {
-schemes.put(StandardScheme.class, new IndexStandardSchemeFactory());
-schemes.put(TupleScheme.class, new IndexTupleSchemeFactory());
- }
-
- private String indexName; // required
- private String indexHandlerClass; // required
- private String dbName; // required
- private String origTableName; // required
- private int createTime; // required
- private int lastAccessTime; // required
- private String indexTableName; // required
- private StorageDescriptor sd; // required
- private Map parameters; // required
- private boolean deferredRebuild; // required
-
- /** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
- public enum _Fields implements org.apache.thrift.TFieldIdEnum {
-INDEX_NAME((short)1, "i
[01/22] hive git commit: HIVE-17626: Query reoptimization using cached runtime statistics (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master cd8eda856 -> 7c22d74c8 http://git-wip-us.apache.org/repos/asf/hive/blob/31e36f01/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 75a2908..2079233 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -214,6 +214,13 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 2100 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_5: 1 + RECORDS_OUT_OPERATOR_GBY_3: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_2: 2100 + RECORDS_OUT_OPERATOR_SEL_1: 2100 + RECORDS_OUT_OPERATOR_TS_0: 2100 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 262144 ALLOCATED_USED_BYTES: 26 @@ -242,6 +249,9 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 Stage-1 INPUT COUNTERS: GROUPED_INPUT_SPLITS_Map_1: 0 INPUT_DIRECTORIES_Map_1: 1 @@ -264,6 +274,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 8 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 8 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 8 + RECORDS_OUT_OPERATOR_SEL_2: 8 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 1048576 ALLOCATED_USED_BYTES: 2732 @@ -296,6 +314,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 22 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 22 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 22 + RECORDS_OUT_OPERATOR_SEL_2: 22 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -326,6 +352,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 16 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 16 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 16 + RECORDS_OUT_OPERATOR_SEL_2: 16 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -356,6 +390,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 18 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 18 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 18 + RECORDS_OUT_OPERATOR_SEL_2: 18 + RECORDS_OUT_OPERATOR_TS_0: 2000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -386,6 +428,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 1 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 1 + RECORDS_OUT_OPERATOR_SEL_2: 1 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -416,6 +466,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 32 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 32 + RECORDS_OUT_OPERATOR_SEL_2: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -446,6 +504,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 32 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 32 + RECORDS_OUT_OPERATOR_FS_6: 1 + RECORDS_OUT_OPERATOR_GBY_4: 1 + RECORDS_OUT_OPERATOR_MAP_0: 0 + RECORDS_OUT_OPERATOR_RS_3: 32 + RECORDS_OUT_OPERATOR_SEL_2: 32 + RECORDS_OUT_OPERATOR_TS_0: 1000 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 1079 CACHE_MISS_BYTES: 0 @@ -476,6 +542,14 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 RECORDS_OUT_INTERMEDIATE_Map_1: 1697 + RECORDS_OUT_INTERMEDIATE_Reducer_2: 0 + RECORDS_OUT_OPERATOR_FIL_7: 16
[14/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
index 9a97d6b..8f3b848 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java
@@ -214,18 +214,6 @@ import org.slf4j.LoggerFactory;
public boolean isPartitionMarkedForEvent(String db_name, String tbl_name,
Map part_vals, PartitionEventType eventType) throws
MetaException, NoSuchObjectException, UnknownDBException,
UnknownTableException, UnknownPartitionException, InvalidPartitionException,
org.apache.thrift.TException;
-public Index add_index(Index new_index, Table index_table) throws
InvalidObjectException, AlreadyExistsException, MetaException,
org.apache.thrift.TException;
-
-public void alter_index(String dbname, String base_tbl_name, String
idx_name, Index new_idx) throws InvalidOperationException, MetaException,
org.apache.thrift.TException;
-
-public boolean drop_index_by_name(String db_name, String tbl_name, String
index_name, boolean deleteData) throws NoSuchObjectException, MetaException,
org.apache.thrift.TException;
-
-public Index get_index_by_name(String db_name, String tbl_name, String
index_name) throws MetaException, NoSuchObjectException,
org.apache.thrift.TException;
-
-public List get_indexes(String db_name, String tbl_name, short
max_indexes) throws NoSuchObjectException, MetaException,
org.apache.thrift.TException;
-
-public List get_index_names(String db_name, String tbl_name, short
max_indexes) throws MetaException, org.apache.thrift.TException;
-
public PrimaryKeysResponse get_primary_keys(PrimaryKeysRequest request)
throws MetaException, NoSuchObjectException, org.apache.thrift.TException;
public ForeignKeysResponse get_foreign_keys(ForeignKeysRequest request)
throws MetaException, NoSuchObjectException, org.apache.thrift.TException;
@@ -590,18 +578,6 @@ import org.slf4j.LoggerFactory;
public void isPartitionMarkedForEvent(String db_name, String tbl_name,
Map part_vals, PartitionEventType eventType,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
-public void add_index(Index new_index, Table index_table,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
-
-public void alter_index(String dbname, String base_tbl_name, String
idx_name, Index new_idx, org.apache.thrift.async.AsyncMethodCallback
resultHandler) throws org.apache.thrift.TException;
-
-public void drop_index_by_name(String db_name, String tbl_name, String
index_name, boolean deleteData, org.apache.thrift.async.AsyncMethodCallback
resultHandler) throws org.apache.thrift.TException;
-
-public void get_index_by_name(String db_name, String tbl_name, String
index_name, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
-
-public void get_indexes(String db_name, String tbl_name, short
max_indexes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
-
-public void get_index_names(String db_name, String tbl_name, short
max_indexes, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
-
public void get_primary_keys(PrimaryKeysRequest request,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
public void get_foreign_keys(ForeignKeysRequest request,
org.apache.thrift.async.AsyncMethodCallback resultHandler) throws
org.apache.thrift.TException;
@@ -3426,190 +3402,6 @@ import org.slf4j.LoggerFactory;
throw new
org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT,
"isPartitionMarkedForEvent failed: unknown result");
}
-public Index add_index(Index new_index, Table index_table) throws
InvalidObjectException, AlreadyExistsException, MetaException,
org.apache.thrift.TException
-{
- send_add_index(new_index, index_table);
- return recv_add_index();
-}
-
-public void send_add_index(Index new_index, Table index_table) throws
org.apache.thrift.TException
-{
- add_index_args args = new add_index_args();
- args.setNew_index(new_index);
- args.setIndex_table(index_table);
- sendBase("add_index", args);
-}
-
-public Index recv_add_index() throws InvalidObjectException,
AlreadyExistsEx
[19/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index cdcde51..244b7ab 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -6443,296 +6443,6 @@ void PartitionSpec::printTo(std::ostream& out) const {
}
-Index::~Index() throw() {
-}
-
-
-void Index::__set_indexName(const std::string& val) {
- this->indexName = val;
-}
-
-void Index::__set_indexHandlerClass(const std::string& val) {
- this->indexHandlerClass = val;
-}
-
-void Index::__set_dbName(const std::string& val) {
- this->dbName = val;
-}
-
-void Index::__set_origTableName(const std::string& val) {
- this->origTableName = val;
-}
-
-void Index::__set_createTime(const int32_t val) {
- this->createTime = val;
-}
-
-void Index::__set_lastAccessTime(const int32_t val) {
- this->lastAccessTime = val;
-}
-
-void Index::__set_indexTableName(const std::string& val) {
- this->indexTableName = val;
-}
-
-void Index::__set_sd(const StorageDescriptor& val) {
- this->sd = val;
-}
-
-void Index::__set_parameters(const std::map & val) {
- this->parameters = val;
-}
-
-void Index::__set_deferredRebuild(const bool val) {
- this->deferredRebuild = val;
-}
-
-uint32_t Index::read(::apache::thrift::protocol::TProtocol* iprot) {
-
- apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
- uint32_t xfer = 0;
- std::string fname;
- ::apache::thrift::protocol::TType ftype;
- int16_t fid;
-
- xfer += iprot->readStructBegin(fname);
-
- using ::apache::thrift::protocol::TProtocolException;
-
-
- while (true)
- {
-xfer += iprot->readFieldBegin(fname, ftype, fid);
-if (ftype == ::apache::thrift::protocol::T_STOP) {
- break;
-}
-switch (fid)
-{
- case 1:
-if (ftype == ::apache::thrift::protocol::T_STRING) {
- xfer += iprot->readString(this->indexName);
- this->__isset.indexName = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 2:
-if (ftype == ::apache::thrift::protocol::T_STRING) {
- xfer += iprot->readString(this->indexHandlerClass);
- this->__isset.indexHandlerClass = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 3:
-if (ftype == ::apache::thrift::protocol::T_STRING) {
- xfer += iprot->readString(this->dbName);
- this->__isset.dbName = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 4:
-if (ftype == ::apache::thrift::protocol::T_STRING) {
- xfer += iprot->readString(this->origTableName);
- this->__isset.origTableName = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 5:
-if (ftype == ::apache::thrift::protocol::T_I32) {
- xfer += iprot->readI32(this->createTime);
- this->__isset.createTime = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 6:
-if (ftype == ::apache::thrift::protocol::T_I32) {
- xfer += iprot->readI32(this->lastAccessTime);
- this->__isset.lastAccessTime = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 7:
-if (ftype == ::apache::thrift::protocol::T_STRING) {
- xfer += iprot->readString(this->indexTableName);
- this->__isset.indexTableName = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 8:
-if (ftype == ::apache::thrift::protocol::T_STRUCT) {
- xfer += this->sd.read(iprot);
- this->__isset.sd = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
- case 9:
-if (ftype == ::apache::thrift::protocol::T_MAP) {
- {
-this->parameters.clear();
-uint32_t _size279;
-::apache::thrift::protocol::TType _ktype280;
-::apache::thrift::protocol::TType _vtype281;
-xfer += iprot->readMapBegin(_ktype280, _vtype281, _size279);
-uint32_t _i283;
-for (_i283 = 0; _i283 < _size279; ++_i283)
-{
- std::string _key284;
- xfer += iprot->readString(_key284);
- std::string& _val285 = this->parameters[_key284];
- xfer += iprot->readString(_val285);
-}
-xfer += iprot->readMapEnd();
- }
- this->__isset.parameters = true;
-} else {
- xfer += iprot->skip(ftype);
-}
-break;
-
[13/22] hive git commit: HIVE-18715: Remove index support from metastore (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UniqueConstraintsResponse.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UniqueConstraintsResponse.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UniqueConstraintsResponse.java
index d545e66..4d45bee 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UniqueConstraintsResponse.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/UniqueConstraintsResponse.java
@@ -354,14 +354,14 @@ import org.slf4j.LoggerFactory;
case 1: // UNIQUE_CONSTRAINTS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
-org.apache.thrift.protocol.TList _list338 =
iprot.readListBegin();
-struct.uniqueConstraints = new
ArrayList(_list338.size);
-SQLUniqueConstraint _elem339;
-for (int _i340 = 0; _i340 < _list338.size; ++_i340)
+org.apache.thrift.protocol.TList _list328 =
iprot.readListBegin();
+struct.uniqueConstraints = new
ArrayList(_list328.size);
+SQLUniqueConstraint _elem329;
+for (int _i330 = 0; _i330 < _list328.size; ++_i330)
{
- _elem339 = new SQLUniqueConstraint();
- _elem339.read(iprot);
- struct.uniqueConstraints.add(_elem339);
+ _elem329 = new SQLUniqueConstraint();
+ _elem329.read(iprot);
+ struct.uniqueConstraints.add(_elem329);
}
iprot.readListEnd();
}
@@ -387,9 +387,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(UNIQUE_CONSTRAINTS_FIELD_DESC);
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
struct.uniqueConstraints.size()));
- for (SQLUniqueConstraint _iter341 : struct.uniqueConstraints)
+ for (SQLUniqueConstraint _iter331 : struct.uniqueConstraints)
{
-_iter341.write(oprot);
+_iter331.write(oprot);
}
oprot.writeListEnd();
}
@@ -414,9 +414,9 @@ import org.slf4j.LoggerFactory;
TTupleProtocol oprot = (TTupleProtocol) prot;
{
oprot.writeI32(struct.uniqueConstraints.size());
-for (SQLUniqueConstraint _iter342 : struct.uniqueConstraints)
+for (SQLUniqueConstraint _iter332 : struct.uniqueConstraints)
{
- _iter342.write(oprot);
+ _iter332.write(oprot);
}
}
}
@@ -425,14 +425,14 @@ import org.slf4j.LoggerFactory;
public void read(org.apache.thrift.protocol.TProtocol prot,
UniqueConstraintsResponse struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
{
-org.apache.thrift.protocol.TList _list343 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
iprot.readI32());
-struct.uniqueConstraints = new
ArrayList(_list343.size);
-SQLUniqueConstraint _elem344;
-for (int _i345 = 0; _i345 < _list343.size; ++_i345)
+org.apache.thrift.protocol.TList _list333 = new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT,
iprot.readI32());
+struct.uniqueConstraints = new
ArrayList(_list333.size);
+SQLUniqueConstraint _elem334;
+for (int _i335 = 0; _i335 < _list333.size; ++_i335)
{
- _elem344 = new SQLUniqueConstraint();
- _elem344.read(iprot);
- struct.uniqueConstraints.add(_elem344);
+ _elem334 = new SQLUniqueConstraint();
+ _elem334.read(iprot);
+ struct.uniqueConstraints.add(_elem334);
}
}
struct.setUniqueConstraintsIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/7c22d74c/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
--
diff --git
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
index dedbcce..2fc0e00 100644
---
a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
+++
b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/WMFullResourcePlan.java
@@ -755,14 +755,14 @@ import org.slf4j.LoggerFactory;
case 2: // POOLS
if (schemeField.type =
[02/22] hive git commit: HIVE-17626: Query reoptimization using cached runtime statistics (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/31e36f01/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java
new file mode 100644
index 000..6b5fa1f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/OperatorStatsReaderHook.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.stats;
+
+import java.util.List;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.tez.TezTask;
+import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.hooks.HookContext.HookType;
+import org.apache.hadoop.hive.ql.hooks.PrivateHookContext;
+import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.tez.common.counters.TezCounter;
+import org.apache.tez.common.counters.TezCounters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OperatorStatsReaderHook implements ExecuteWithHookContext {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(OperatorStatsReaderHook.class);
+ private boolean collectOnSuccess;
+
+ @Override
+ public void run(HookContext hookContext) throws Exception {
+
+if (hookContext.getHookType() == HookType.PRE_EXEC_HOOK) {
+ return;
+}
+if (hookContext.getHookType() == HookType.POST_EXEC_HOOK &&
!isCollectOnSuccess()) {
+ return;
+}
+
+HiveConf conf = hookContext.getConf();
+QueryPlan plan = hookContext.getQueryPlan();
+List rootTasks = Utilities.getTezTasks(plan.getRootTasks());
+for (TezTask tezTask : rootTasks) {
+ List baseWorks = tezTask.getWork().getAllWork();
+ for (BaseWork baseWork : baseWorks) {
+String vertexName = baseWork.getName();
+LOG.debug("Reading runtime statistics for tez vertex task: {}",
vertexName);
+TezCounters counters = tezTask.getTezCounters();
+if (counters != null) {
+ String groupName = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVECOUNTERGROUP);
+ for (Operator op :
baseWork.getAllOperators()) {
+String operatorId = op.getOperatorId();
+OperatorStats operatorStats = null;
+String counterName =
Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId;
+TezCounter tezCounter =
counters.getGroup(groupName).findCounter(counterName, false);
+if (tezCounter != null) {
+ if (operatorStats == null) {
+operatorStats = new OperatorStats(operatorId);
+ }
+ operatorStats.setOutputRecords(tezCounter.getValue());
+}
+
+if (operatorStats != null) {
+ ((PrivateHookContext)
hookContext).getContext().getPlanMapper().link(op, operatorStats);
+} else {
+ LOG.debug("Unable to get statistics for vertex: {} opId: {}
groupName: {}", vertexName, operatorId,
+ groupName);
+}
+ }
+}
+ }
+}
+ }
+
+ public boolean isCollectOnSuccess() {
+return collectOnSuccess;
+ }
+
+ public void setCollectOnSuccess(boolean collectOnSuccess) {
+this.collectOnSuccess = collectOnSuccess;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/31e36f01/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java
new file mode 100644
index 000..c5c7383
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrueOOM.java
@@ -0,0 +1
[03/22] hive git commit: HIVE-17626: Query reoptimization using cached runtime statistics (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/31e36f01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 9a3f81c..22b052c 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -27,11 +27,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
import java.util.Stack;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
@@ -74,6 +75,8 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.Statistics.State;
+import org.apache.hadoop.hive.ql.plan.mapper.RuntimeStatsSource;
+import org.apache.hadoop.hive.ql.stats.OperatorStats;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -132,7 +135,9 @@ public class StatsRulesProcFactory {
try {
// gather statistics for the first time and the attach it to table
scan operator
Statistics stats = StatsUtils.collectStatistics(aspCtx.getConf(),
partList, colStatsCached, table, tsop);
-tsop.setStatistics(stats.clone());
+
+stats = applyRuntimeStats(aspCtx.getParseContext().getContext(),
stats, (Operator) tsop);
+tsop.setStatistics(stats);
if (LOG.isDebugEnabled()) {
LOG.debug("[0] STATS-" + tsop.toString() + " (" +
table.getTableName() + "): " +
@@ -144,6 +149,7 @@ public class StatsRulesProcFactory {
}
return null;
}
+
}
/**
@@ -181,14 +187,15 @@ public class StatsRulesProcFactory {
if (satisfyPrecondition(parentStats)) {
// this will take care of mapping between input column names and
output column names. The
// returned column stats will have the output column names.
-List colStats =
StatsUtils.getColStatisticsFromExprMap(conf, parentStats,
-sop.getColumnExprMap(), sop.getSchema());
+List colStats =
+StatsUtils.getColStatisticsFromExprMap(conf, parentStats,
sop.getColumnExprMap(), sop.getSchema());
stats.setColumnStats(colStats);
// in case of select(*) the data size does not change
if (!sop.getConf().isSelectStar() &&
!sop.getConf().isSelStarNoCompute()) {
long dataSize =
StatsUtils.getDataSizeFromColumnStats(stats.getNumRows(), colStats);
stats.setDataSize(dataSize);
}
+stats = applyRuntimeStats(aspCtx.getParseContext().getContext(),
stats, (Operator) sop);
sop.setStatistics(stats);
if (LOG.isDebugEnabled()) {
@@ -196,7 +203,8 @@ public class StatsRulesProcFactory {
}
} else {
if (parentStats != null) {
- sop.setStatistics(parentStats.clone());
+ stats = applyRuntimeStats(aspCtx.getParseContext().getContext(),
stats, (Operator) sop);
+ sop.setStatistics(stats);
if (LOG.isDebugEnabled()) {
LOG.debug("[1] STATS-" + sop.toString() + ": " +
parentStats.extendedToString());
@@ -299,7 +307,10 @@ public class StatsRulesProcFactory {
LOG.debug("[1] STATS-" + fop.toString() + ": " +
st.extendedToString());
}
}
+
+st = applyRuntimeStats(aspCtx.getParseContext().getContext(), st,
(Operator) fop);
fop.setStatistics(st);
+
aspCtx.setAndExprStats(null);
}
return null;
@@ -1249,6 +1260,7 @@ public class StatsRulesProcFactory {
}
}
+ stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats,
(Operator) gop);
gop.setStatistics(stats);
if (LOG.isDebugEnabled() && stats != null) {
@@ -1576,6 +1588,7 @@ public class StatsRulesProcFactory {
}
}
+stats = applyRuntimeStats(aspCtx.getParseContext().getContext(),
stats, jop);
jop.setStatistics(stats);
if (LOG.isDebugEnabled()) {
@@ -1665,6 +1678,7 @@ public class StatsRulesProcFactory {
}
}
+wcStats = applyRuntimeStats(aspCtx.getParseContext().getContext(),
wcStats, jop);
jop.setStatistics(wcStats);
