hive git commit: HIVE-19130: NPE is thrown when REPL LOAD applied drop partition event (Sankar Hariappan, reviewed by Mahesh Kumar Behera, Thejas M Nair)
Repository: hive Updated Branches: refs/heads/master 244ca8e5c -> d9fae0493 HIVE-19130: NPE is thrown when REPL LOAD applied drop partition event (Sankar Hariappan, reviewed by Mahesh Kumar Behera, Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d9fae049 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d9fae049 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d9fae049 Branch: refs/heads/master Commit: d9fae049305e20ec8a72e581a2fc938028523402 Parents: 244ca8e Author: Sankar HariappanAuthored: Thu Apr 12 10:29:47 2018 +0530 Committer: Sankar Hariappan Committed: Thu Apr 12 10:29:47 2018 +0530 -- ...TestReplicationScenariosAcrossInstances.java | 55 .../hadoop/hive/ql/parse/WarehouseInstance.java | 5 ++ .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 8 ++- .../apache/hadoop/hive/ql/parse/EximUtil.java | 10 .../hive/ql/parse/ImportSemanticAnalyzer.java | 31 +-- .../parse/repl/load/message/TableHandler.java | 1 + 6 files changed, 106 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/d9fae049/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java -- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 689ca76..70e1aa7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -620,4 +620,59 @@ public class TestReplicationScenariosAcrossInstances { .run("show functions like '" + replicatedDbName + "*'") .verifyResult(null); } + + @Test + public void testIncrementalReplWithEventsBatchHavingDropCreateTable() throws Throwable { +// Bootstrap dump with empty db +WarehouseInstance.Tuple bootstrapTuple = primary.dump(primaryDbName, null); + +// Bootstrap load in replica +replica.load(replicatedDbName, bootstrapTuple.dumpLocation) +.status(replicatedDbName) +.verifyResult(bootstrapTuple.lastReplicationId); + +// First incremental dump +WarehouseInstance.Tuple firstIncremental = primary.run("use " + primaryDbName) +.run("create table table1 (i int)") +.run("create table table2 (id int) partitioned by (country string)") +.run("insert into table1 values (1)") +.run("insert into table2 partition(country='india') values(1)") +.dump(primaryDbName, bootstrapTuple.lastReplicationId); + +// Second incremental dump +WarehouseInstance.Tuple secondIncremental = primary.run("use " + primaryDbName) +.run("drop table table1") +.run("drop table table2") +.run("create table table2 (id int) partitioned by (country string)") +.run("alter table table2 add partition(country='india')") +.run("alter table table2 drop partition(country='india')") +.run("insert into table2 partition(country='us') values(2)") +.run("create table table1 (i int)") +.run("insert into table1 values (2)") +.dump(primaryDbName, firstIncremental.lastReplicationId); + +// First incremental load +replica.load(replicatedDbName, firstIncremental.dumpLocation) +.status(replicatedDbName) +.verifyResult(firstIncremental.lastReplicationId) +.run("use " + replicatedDbName) +.run("show tables") +.verifyResults(new String[] {"table1", "table2"}) +.run("select * from table1") +.verifyResults(new String[] {"1"}) +.run("select id from table2 order by id") +.verifyResults(new String[] {"1"}); + +// Second incremental load +replica.load(replicatedDbName, secondIncremental.dumpLocation) +.status(replicatedDbName) +.verifyResult(secondIncremental.lastReplicationId) +.run("use " + replicatedDbName) +.run("show tables") +.verifyResults(new String[] {"table1", "table2"}) +.run("select * from table1") +.verifyResults(new String[] {"2"}) +.run("select id from table2 order by id") +.verifyResults(new String[] {"2"}); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/d9fae049/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
[11/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query29.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query29.q.out b/ql/src/test/results/clientpositive/perf/spark/query29.q.out index 456fce7..530a625 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query29.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query29.q.out @@ -124,7 +124,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 14 +Map 13 Map Operator Tree: TableScan alias: d2 @@ -146,51 +146,51 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Map 12 (PARTITION-LEVEL SORT, 306), Map 9 (PARTITION-LEVEL SORT, 306) -Reducer 11 <- Map 13 (PARTITION-LEVEL SORT, 374), Reducer 10 (PARTITION-LEVEL SORT, 374) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398) -Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 442), Reducer 2 (PARTITION-LEVEL SORT, 442) -Reducer 4 <- Reducer 11 (PARTITION-LEVEL SORT, 850), Reducer 3 (PARTITION-LEVEL SORT, 850) -Reducer 5 <- Reducer 4 (GROUP, 582) -Reducer 6 <- Reducer 5 (SORT, 1) +Reducer 10 <- Map 14 (PARTITION-LEVEL SORT, 486), Reducer 9 (PARTITION-LEVEL SORT, 486) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 6 (PARTITION-LEVEL SORT, 306) +Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 917), Reducer 2 (PARTITION-LEVEL SORT, 917) +Reducer 4 <- Reducer 3 (GROUP, 640) +Reducer 5 <- Reducer 4 (SORT, 1) +Reducer 8 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL SORT, 398) +Reducer 9 <- Map 12 (PARTITION-LEVEL SORT, 476), Reducer 8 (PARTITION-LEVEL SORT, 476) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: store_sales - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE + alias: catalog_sales + Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean) -Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE +predicate: (cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) (type: boolean) +Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE + expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) -Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE -value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int) -Map 12 +Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE +value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int) +Map 11 Map Operator Tree: TableScan - alias: d3 + alias: d1 Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ((d_year) IN (1999, 2000, 2001) and d_date_sk is not null) (type: boolean) -Statistics: Num rows: 36525 Data size: 40871475 Basic stats: COMPLETE Column stats: NONE +predicate:
[09/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query45.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query45.q.out b/ql/src/test/results/clientpositive/perf/spark/query45.q.out index 7e1cc88..b674400 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query45.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query45.q.out @@ -46,10 +46,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 6 <- Map 5 (GROUP, 1) +Reducer 16 <- Map 15 (GROUP, 1) A masked pattern was here Vertices: -Map 5 +Map 15 Map Operator Tree: TableScan alias: item @@ -70,7 +70,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint), _col1 (type: bigint) -Reducer 6 +Reducer 16 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -88,45 +88,33 @@ STAGE PLANS: Spark Edges: Reducer 11 <- Map 10 (GROUP, 3) -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 15 (PARTITION-LEVEL SORT, 154) -Reducer 14 <- Map 16 (PARTITION-LEVEL SORT, 706), Reducer 13 (PARTITION-LEVEL SORT, 706) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 527), Reducer 9 (PARTITION-LEVEL SORT, 527) -Reducer 3 <- Reducer 2 (GROUP, 224) -Reducer 4 <- Reducer 3 (SORT, 1) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 14 (PARTITION-LEVEL SORT, 154) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 6 (PARTITION-LEVEL SORT, 855) +Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 777), Reducer 9 (PARTITION-LEVEL SORT, 777) +Reducer 4 <- Reducer 3 (GROUP, 230) +Reducer 5 <- Reducer 4 (SORT, 1) Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 7), Reducer 11 (PARTITION-LEVEL SORT, 7) -Reducer 9 <- Reducer 14 (PARTITION-LEVEL SORT, 191), Reducer 8 (PARTITION-LEVEL SORT, 191) +Reducer 9 <- Reducer 13 (PARTITION-LEVEL SORT, 174), Reducer 8 (PARTITION-LEVEL SORT, 174) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: customer_address - Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE + alias: customer + Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ca_address_sk is not null (type: boolean) -Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE +predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) +Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ca_address_sk (type: int), ca_county (type: string), ca_zip (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 - 1 -outputColumnNames: _col0, _col1, _col2, _col3, _col4 -input vertices: - 1 Reducer 6 -Statistics: Num rows: 4000 Data size: 41275195284 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col0 (type: int) - sort order: + - Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4000 Data size: 41275195284 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint), _col4 (type: bigint) -Local Work: - Map Reduce Local Work + expressions: c_customer_sk (type: int), c_current_addr_sk (type: int) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col1
[05/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query75.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out b/ql/src/test/results/clientpositive/perf/spark/query75.q.out index f4fca34..4b76570 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out @@ -144,25 +144,25 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 400), Map 16 (PARTITION-LEVEL SORT, 400) -Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 438), Reducer 13 (PARTITION-LEVEL SORT, 438) +Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 398), Map 16 (PARTITION-LEVEL SORT, 398) +Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 440), Reducer 13 (PARTITION-LEVEL SORT, 440) Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 516), Reducer 14 (PARTITION-LEVEL SORT, 516) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 16 (PARTITION-LEVEL SORT, 308) -Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 156), Map 19 (PARTITION-LEVEL SORT, 156) -Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 169), Reducer 20 (PARTITION-LEVEL SORT, 169) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 16 (PARTITION-LEVEL SORT, 306) +Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 154), Map 19 (PARTITION-LEVEL SORT, 154) +Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 171), Reducer 20 (PARTITION-LEVEL SORT, 171) Reducer 22 <- Map 25 (PARTITION-LEVEL SORT, 196), Reducer 21 (PARTITION-LEVEL SORT, 196) -Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 308), Map 32 (PARTITION-LEVEL SORT, 308) -Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 336), Reducer 27 (PARTITION-LEVEL SORT, 336) +Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 306), Map 32 (PARTITION-LEVEL SORT, 306) +Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 338), Reducer 27 (PARTITION-LEVEL SORT, 338) Reducer 29 <- Map 34 (PARTITION-LEVEL SORT, 393), Reducer 28 (PARTITION-LEVEL SORT, 393) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 336), Reducer 2 (PARTITION-LEVEL SORT, 336) +Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 338), Reducer 2 (PARTITION-LEVEL SORT, 338) Reducer 30 <- Reducer 29 (GROUP, 934), Reducer 38 (GROUP, 934) Reducer 31 <- Reducer 30 (GROUP PARTITION-LEVEL SORT, 671), Reducer 45 (GROUP PARTITION-LEVEL SORT, 671) -Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 400), Map 35 (PARTITION-LEVEL SORT, 400) -Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 438), Reducer 36 (PARTITION-LEVEL SORT, 438) +Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 398), Map 35 (PARTITION-LEVEL SORT, 398) +Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 440), Reducer 36 (PARTITION-LEVEL SORT, 440) Reducer 38 <- Map 41 (PARTITION-LEVEL SORT, 516), Reducer 37 (PARTITION-LEVEL SORT, 516) Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 393), Reducer 3 (PARTITION-LEVEL SORT, 393) -Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 156), Map 42 (PARTITION-LEVEL SORT, 156) -Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 169), Reducer 43 (PARTITION-LEVEL SORT, 169) +Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 154), Map 42 (PARTITION-LEVEL SORT, 154) +Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 171), Reducer 43 (PARTITION-LEVEL SORT, 171) Reducer 45 <- Map 48 (PARTITION-LEVEL SORT, 196), Reducer 44 (PARTITION-LEVEL SORT, 196) Reducer 5 <- Reducer 15 (GROUP, 934), Reducer 4 (GROUP, 934) Reducer 6 <- Reducer 22 (GROUP PARTITION-LEVEL SORT, 671), Reducer 5 (GROUP PARTITION-LEVEL SORT, 671) @@ -183,28 +183,29 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col1 (type: int) +key expressions: _col0 (type: int) sort order: + -Map-reduce partition columns: _col1 (type: int) +Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) +value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: date_dim - Statistics: Num rows: 73049 Data size: 81741831 Basic
[10/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query4.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query4.q.out b/ql/src/test/results/clientpositive/perf/spark/query4.q.out index 5074d49..75c4fbb 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query4.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query4.q.out @@ -225,20 +225,20 @@ STAGE PLANS: Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 398), Map 9 (PARTITION-LEVEL SORT, 398) Reducer 11 <- Map 14 (PARTITION-LEVEL SORT, 975), Reducer 10 (PARTITION-LEVEL SORT, 975) Reducer 12 <- Reducer 11 (GROUP, 481) -Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 306), Map 19 (PARTITION-LEVEL SORT, 306) -Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 873), Reducer 16 (PARTITION-LEVEL SORT, 873) -Reducer 18 <- Reducer 17 (GROUP, 369) +Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 398), Map 19 (PARTITION-LEVEL SORT, 398) +Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 975), Reducer 16 (PARTITION-LEVEL SORT, 975) +Reducer 18 <- Reducer 17 (GROUP, 481) Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 7 (PARTITION-LEVEL SORT, 306) -Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 154), Map 25 (PARTITION-LEVEL SORT, 154) -Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 706), Reducer 22 (PARTITION-LEVEL SORT, 706) -Reducer 24 <- Reducer 23 (GROUP, 186) +Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 306), Map 25 (PARTITION-LEVEL SORT, 306) +Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 873), Reducer 22 (PARTITION-LEVEL SORT, 873) +Reducer 24 <- Reducer 23 (GROUP, 369) Reducer 28 <- Map 27 (PARTITION-LEVEL SORT, 154), Map 31 (PARTITION-LEVEL SORT, 154) Reducer 29 <- Map 32 (PARTITION-LEVEL SORT, 706), Reducer 28 (PARTITION-LEVEL SORT, 706) Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 873), Reducer 2 (PARTITION-LEVEL SORT, 873) Reducer 30 <- Reducer 29 (GROUP, 186) -Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 398), Map 37 (PARTITION-LEVEL SORT, 398) -Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 975), Reducer 34 (PARTITION-LEVEL SORT, 975) -Reducer 36 <- Reducer 35 (GROUP, 481) +Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 154), Map 37 (PARTITION-LEVEL SORT, 154) +Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 706), Reducer 34 (PARTITION-LEVEL SORT, 706) +Reducer 36 <- Reducer 35 (GROUP, 186) Reducer 4 <- Reducer 3 (GROUP, 369) Reducer 5 <- Reducer 12 (PARTITION-LEVEL SORT, 690), Reducer 18 (PARTITION-LEVEL SORT, 690), Reducer 24 (PARTITION-LEVEL SORT, 690), Reducer 30 (PARTITION-LEVEL SORT, 690), Reducer 36 (PARTITION-LEVEL SORT, 690), Reducer 4 (PARTITION-LEVEL SORT, 690) Reducer 6 <- Reducer 5 (SORT, 1) @@ -300,20 +300,20 @@ STAGE PLANS: Map 15 Map Operator Tree: TableScan - alias: catalog_sales - Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE + alias: store_sales + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (cs_bill_customer_sk is not null and cs_sold_date_sk is not null) (type: boolean) -Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE +predicate: (ss_customer_sk is not null and ss_sold_date_sk is not null) (type: boolean) +Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type: int), cs_ext_discount_amt (type: decimal(7,2)), cs_ext_sales_price (type: decimal(7,2)), cs_ext_wholesale_cost (type: decimal(7,2)), cs_ext_list_price (type: decimal(7,2)) + expressions: ss_sold_date_sk (type: int), ss_customer_sk (type: int), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price (type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), ss_ext_list_price (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 - Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0
[03/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query81.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query81.q.out b/ql/src/test/results/clientpositive/perf/spark/query81.q.out index 23998d6..5b2d5b3 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query81.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query81.q.out @@ -66,37 +66,72 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 25), Map 14 (PARTITION-LEVEL SORT, 25) -Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 344), Reducer 11 (PARTITION-LEVEL SORT, 344) -Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349) -Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 (PARTITION-LEVEL SORT, 697) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 25), Map 8 (PARTITION-LEVEL SORT, 25) -Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 344), Reducer 2 (PARTITION-LEVEL SORT, 344) -Reducer 4 <- Reducer 3 (GROUP, 349) -Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 (PARTITION-LEVEL SORT, 262) -Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 (PARTITION-LEVEL SORT, 656) -Reducer 7 <- Reducer 6 (SORT, 1) +Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 (PARTITION-LEVEL SORT, 262) +Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 25), Map 17 (PARTITION-LEVEL SORT, 25) +Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 344), Reducer 14 (PARTITION-LEVEL SORT, 344) +Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL SORT, 697) +Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 (PARTITION-LEVEL SORT, 656) +Reducer 4 <- Reducer 3 (SORT, 1) +Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 25), Map 6 (PARTITION-LEVEL SORT, 25) +Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 344), Reducer 7 (PARTITION-LEVEL SORT, 344) +Reducer 9 <- Reducer 8 (GROUP, 349) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: catalog_returns - Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE + alias: customer + Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (cr_returned_date_sk is not null and cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) (type: boolean) -Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE +predicate: (c_current_addr_sk is not null and c_customer_sk is not null) (type: boolean) +Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: cr_returned_date_sk (type: int), cr_returning_customer_sk (type: int), cr_returning_addr_sk (type: int), cr_return_amt_inc_tax (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 28798881 Data size: 3057234680 Basic stats: COMPLETE Column stats: NONE + expressions: c_customer_sk (type: int), c_customer_id (type: string), c_current_addr_sk (type: int), c_salutation (type: string), c_first_name (type: string), c_last_name (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col2 (type: int) +sort order: + +Map-reduce partition columns: _col2 (type: int) +Statistics: Num rows: 8000 Data size: 68801615852 Basic stats: COMPLETE Column stats: NONE +value expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string), _col4 (type: string), _col5 (type: string) +Map 11 +Map Operator Tree: +TableScan + alias: date_dim + Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: ((d_year = 1998) and d_date_sk is not null) (type: boolean) +Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE +
[07/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query58.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query58.q.out b/ql/src/test/results/clientpositive/perf/spark/query58.q.out index eb5dffe..f06cbef 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query58.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query58.q.out @@ -1,6 +1,6 @@ -Warning: Map Join MAPJOIN[180][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[181][bigTable=?] in task 'Stage-1:MAPRED' is a cross product -Warning: Map Join MAPJOIN[182][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[183][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[184][bigTable=?] in task 'Stage-1:MAPRED' is a cross product +Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross product PREHOOK: query: explain with ss_items as (select i_item_id item_id @@ -140,10 +140,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: -Reducer 12 <- Map 11 (GROUP, 1) +Reducer 11 <- Map 10 (GROUP, 1) A masked pattern was here Vertices: -Map 11 +Map 10 Map Operator Tree: TableScan alias: date_dim @@ -162,7 +162,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 12 +Reducer 11 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -184,10 +184,10 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: -Reducer 25 <- Map 24 (GROUP, 1) +Reducer 24 <- Map 23 (GROUP, 1) A masked pattern was here Vertices: -Map 24 +Map 23 Map Operator Tree: TableScan alias: date_dim @@ -206,7 +206,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 25 +Reducer 24 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -228,10 +228,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: -Reducer 38 <- Map 37 (GROUP, 1) +Reducer 37 <- Map 36 (GROUP, 1) A masked pattern was here Vertices: -Map 37 +Map 36 Map Operator Tree: TableScan alias: date_dim @@ -250,7 +250,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) -Reducer 38 +Reducer 37 Local Work: Map Reduce Local Work Reduce Operator Tree: @@ -272,47 +272,76 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 10 <- Reducer 9 (GROUP, 2) -Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 2), Reducer 23 (PARTITION-LEVEL SORT, 2) -Reducer 18 <- Map 27 (PARTITION-LEVEL SORT, 398), Reducer 17 (PARTITION-LEVEL SORT, 398) -Reducer 19 <- Map 28 (PARTITION-LEVEL SORT, 442), Reducer 18 (PARTITION-LEVEL SORT, 442) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 10 (PARTITION-LEVEL SORT, 2) -Reducer 20 <- Reducer 19 (GROUP, 481) -Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Map 26 (PARTITION-LEVEL SORT, 2) -Reducer 23 <- Reducer 22 (GROUP, 2) -Reducer 3 <- Map 14 (PARTITION-LEVEL SORT, 306), Reducer 2 (PARTITION-LEVEL SORT, 306) -Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 2), Reducer 36 (PARTITION-LEVEL SORT, 2) -Reducer 31 <- Map 40 (PARTITION-LEVEL SORT, 154), Reducer 30 (PARTITION-LEVEL SORT, 154) -Reducer 32 <- Map 41 (PARTITION-LEVEL SORT, 174), Reducer 31 (PARTITION-LEVEL SORT, 174) -Reducer 33 <- Reducer 32 (GROUP, 186) -Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Map 39 (PARTITION-LEVEL SORT, 2) -Reducer 36 <- Reducer 35 (GROUP, 2) -Reducer 4 <- Map 15 (PARTITION-LEVEL SORT, 341), Reducer 3 (PARTITION-LEVEL SORT, 341) -Reducer 5 <- Reducer 4 (GROUP, 369) -Reducer 6 <- Reducer 20 (PARTITION-LEVEL SORT, 518), Reducer 33 (PARTITION-LEVEL SORT, 518), Reducer 5 (PARTITION-LEVEL SORT, 518) -Reducer 7 <- Reducer 6 (SORT, 1) -Reducer 9 <- Map 13 (PARTITION-LEVEL SORT, 2), Map 8 (PARTITION-LEVEL SORT, 2) +Reducer 13 <- Map 12
[06/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query65.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query65.q.out b/ql/src/test/results/clientpositive/perf/spark/query65.q.out index 575fc5c..3b3baef 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query65.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query65.q.out @@ -57,8 +57,7 @@ POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-2 is a root stage Stage-3 depends on stages: Stage-2 - Stage-4 depends on stages: Stage-3 - Stage-1 depends on stages: Stage-4 + Stage-1 depends on stages: Stage-3 Stage-0 depends on stages: Stage-1 STAGE PLANS: @@ -66,29 +65,6 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 11 -Map Operator Tree: -TableScan - alias: store - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: s_store_sk is not null (type: boolean) -Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: s_store_sk (type: int), s_store_name (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE - Spark HashTable Sink Operator -keys: - 0 _col0 (type: int) - 1 _col0 (type: int) -Local Work: - Map Reduce Local Work - - Stage: Stage-3 -Spark - A masked pattern was here - Vertices: Map 6 Map Operator Tree: TableScan @@ -108,7 +84,7 @@ STAGE PLANS: Local Work: Map Reduce Local Work - Stage: Stage-4 + Stage: Stage-3 Spark A masked pattern was here Vertices: @@ -135,8 +111,8 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 437) -Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) -Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 86), Reducer 3 (PARTITION-LEVEL SORT, 86) +Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 328), Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328) +Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 166), Reducer 3 (PARTITION-LEVEL SORT, 166) Reducer 5 <- Reducer 4 (SORT, 1) Reducer 8 <- Map 7 (GROUP PARTITION-LEVEL SORT, 437) A masked pattern was here @@ -180,6 +156,24 @@ STAGE PLANS: Map 10 Map Operator Tree: TableScan + alias: store + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: s_store_sk is not null (type: boolean) +Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: s_store_sk (type: int), s_store_name (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator +key expressions: _col0 (type: int) +sort order: + +Map-reduce partition columns: _col0 (type: int) +Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE +value expressions: _col1 (type: string) +Map 11 +Map Operator Tree: +TableScan alias: item Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE Column stats: NONE Filter Operator @@ -250,23 +244,23 @@ STAGE PLANS: Join Operator condition map: Inner Join 0 to 1 + Inner Join 0 to 2 keys: 0 _col0 (type: int) 1 _col0 (type: int) -outputColumnNames: _col0, _col1, _col2, _col4 -Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE Column stats: NONE + 2 _col0 (type: int) +outputColumnNames: _col1, _col2, _col4, _col6 +Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE Column stats: NONE Filter Operator
[08/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query50.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query50.q.out b/ql/src/test/results/clientpositive/perf/spark/query50.q.out index b978201..fde0c4e 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query50.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query50.q.out @@ -124,7 +124,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 6 +Map 10 Map Operator Tree: TableScan alias: store @@ -138,7 +138,7 @@ STAGE PLANS: Statistics: Num rows: 1704 Data size: 3256276 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col3 (type: int) + 0 _col10 (type: int) 1 _col0 (type: int) Local Work: Map Reduce Local Work @@ -146,63 +146,16 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 476), Reducer 8 (PARTITION-LEVEL SORT, 476) -Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 481), Reducer 2 (PARTITION-LEVEL SORT, 481) -Reducer 4 <- Reducer 3 (GROUP, 529) -Reducer 5 <- Reducer 4 (SORT, 1) -Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 36), Map 9 (PARTITION-LEVEL SORT, 36) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 36), Map 7 (PARTITION-LEVEL SORT, 36) +Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 436), Reducer 2 (PARTITION-LEVEL SORT, 436) +Reducer 4 <- Map 9 (PARTITION-LEVEL SORT, 438), Reducer 3 (PARTITION-LEVEL SORT, 438) +Reducer 5 <- Reducer 4 (GROUP, 529) +Reducer 6 <- Reducer 5 (SORT, 1) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: store_sales - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: (ss_customer_sk is not null and ss_item_sk is not null and ss_sold_date_sk is not null and ss_store_sk is not null and ss_ticket_number is not null) (type: boolean) -Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_store_sk (type: int), ss_ticket_number (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE - Map Join Operator -condition map: - Inner Join 0 to 1 -keys: - 0 _col3 (type: int) - 1 _col0 (type: int) -outputColumnNames: _col0, _col1, _col2, _col4, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15 -input vertices: - 1 Map 6 -Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE -Reduce Output Operator - key expressions: _col1 (type: int), _col2 (type: int), _col4 (type: int) - sort order: +++ - Map-reduce partition columns: _col1 (type: int), _col2 (type: int), _col4 (type: int) - Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col6 (type: string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 (type: string), _col11 (type: string), _col12 (type: string), _col13 (type: string), _col14 (type: string), _col15 (type: string) -Local Work: - Map Reduce Local Work -Map 10 -Map Operator Tree: -TableScan - alias: d1 - Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE - Filter Operator -predicate: d_date_sk is not null (type: boolean) -Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: d_date_sk (type: int) - outputColumnNames: _col0 - Statistics: Num rows:
[02/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query85.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out b/ql/src/test/results/clientpositive/perf/spark/query85.q.out index c82dcf7..d60751c 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out @@ -175,7 +175,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 15 +Map 13 Map Operator Tree: TableScan alias: reason @@ -212,21 +212,21 @@ STAGE PLANS: Statistics: Num rows: 4602 Data size: 2696178 Basic stats: COMPLETE Column stats: NONE Spark HashTable Sink Operator keys: - 0 _col0 (type: int) - 1 _col2 (type: int) + 0 _col10 (type: int) + 1 _col0 (type: int) Local Work: Map Reduce Local Work Stage: Stage-1 Spark Edges: -Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 20), Map 14 (PARTITION-LEVEL SORT, 20) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Map 8 (PARTITION-LEVEL SORT, 90) -Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 17), Reducer 2 (PARTITION-LEVEL SORT, 17) -Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 19), Reducer 3 (PARTITION-LEVEL SORT, 19) -Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 35), Reducer 4 (PARTITION-LEVEL SORT, 35) -Reducer 6 <- Reducer 5 (GROUP, 2) -Reducer 7 <- Reducer 6 (SORT, 1) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 28), Map 9 (PARTITION-LEVEL SORT, 28) +Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 98), Reducer 2 (PARTITION-LEVEL SORT, 98) +Reducer 4 <- Map 12 (PARTITION-LEVEL SORT, 5), Reducer 3 (PARTITION-LEVEL SORT, 5) +Reducer 5 <- Map 14 (PARTITION-LEVEL SORT, 11), Reducer 4 (PARTITION-LEVEL SORT, 11) +Reducer 6 <- Map 15 (PARTITION-LEVEL SORT, 7), Reducer 5 (PARTITION-LEVEL SORT, 7) +Reducer 7 <- Reducer 6 (GROUP, 7) +Reducer 8 <- Reducer 7 (SORT, 1) A masked pattern was here Vertices: Map 1 @@ -242,61 +242,32 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col2 (type: int) -sort order: + -Map-reduce partition columns: _col2 (type: int) +key expressions: _col0 (type: int), _col5 (type: int) +sort order: ++ +Map-reduce partition columns: _col0 (type: int), _col5 (type: int) Statistics: Num rows: 14398467 Data size: 1325194184 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col1 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)) +value expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7 (type: decimal(7,2)) Map 10 Map Operator Tree: TableScan - alias: cd2 - Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE + alias: customer_address + Statistics: Num rows: 4000 Data size: 40595195284 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: (((cd_education_status = '4 yr Degree') or (cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or (cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status is not null and cd_marital_status is not null) (type: boolean) -Statistics: Num rows: 1861800 Data size: 717186159 Basic stats: COMPLETE Column stats: NONE +predicate: ((ca_country = 'United States') and (ca_state) IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is not null) (type: boolean) +Statistics: Num rows: 1000 Data size: 10148798821 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: cd_demo_sk (type: int), cd_marital_status (type: string), cd_education_status (type: string) - outputColumnNames: _col0,
[13/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan) Signed-off-by: Ashutosh ChauhanProject: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/244ca8e5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/244ca8e5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/244ca8e5 Branch: refs/heads/master Commit: 244ca8e5c3192acd017d691ccdbaf0fa06c9fe39 Parents: b3fe652 Author: Zoltan Haindrich Authored: Wed Apr 11 20:05:01 2018 -0700 Committer: Ashutosh Chauhan Committed: Wed Apr 11 20:05:01 2018 -0700 -- data/conf/perf-reg/spark/hive-site.xml | 6 + data/conf/perf-reg/tez/hive-site.xml| 5 + .../hive/cli/control/CorePerfCliDriver.java | 6 +- .../hadoop/hive/ql/MetaStoreDumpUtility.java| 231 + .../org/apache/hadoop/hive/ql/QTestUtil.java| 221 + .../hive/ql/parse/BaseSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- .../clientpositive/perf/spark/query11.q.out | 48 +- .../clientpositive/perf/spark/query15.q.out | 164 +++ .../clientpositive/perf/spark/query16.q.out | 70 +-- .../clientpositive/perf/spark/query18.q.out | 216 .../clientpositive/perf/spark/query19.q.out | 218 - .../clientpositive/perf/spark/query21.q.out | 114 +++-- .../clientpositive/perf/spark/query24.q.out | 282 +-- .../clientpositive/perf/spark/query25.q.out | 118 ++--- .../clientpositive/perf/spark/query29.q.out | 280 +-- .../clientpositive/perf/spark/query30.q.out | 262 +- .../clientpositive/perf/spark/query32.q.out | 132 ++--- .../clientpositive/perf/spark/query34.q.out | 34 +- .../clientpositive/perf/spark/query35.q.out | 74 ++- .../clientpositive/perf/spark/query37.q.out | 16 +- .../clientpositive/perf/spark/query4.q.out | 214 .../clientpositive/perf/spark/query40.q.out | 116 +++-- .../clientpositive/perf/spark/query44.q.out | 246 +- .../clientpositive/perf/spark/query45.q.out | 204 .../clientpositive/perf/spark/query46.q.out | 104 ++-- .../clientpositive/perf/spark/query47.q.out | 92 ++-- .../clientpositive/perf/spark/query48.q.out | 94 ++-- .../clientpositive/perf/spark/query5.q.out | 38 +- .../clientpositive/perf/spark/query50.q.out | 196 .../clientpositive/perf/spark/query53.q.out | 105 ++-- .../clientpositive/perf/spark/query54.q.out | 277 ++- .../clientpositive/perf/spark/query57.q.out | 92 ++-- .../clientpositive/perf/spark/query58.q.out | 490 +-- .../clientpositive/perf/spark/query6.q.out | 350 +++-- .../clientpositive/perf/spark/query61.q.out | 40 +- .../clientpositive/perf/spark/query63.q.out | 105 ++-- .../clientpositive/perf/spark/query65.q.out | 100 ++-- .../clientpositive/perf/spark/query66.q.out | 20 +- .../clientpositive/perf/spark/query67.q.out | 137 +++--- .../clientpositive/perf/spark/query68.q.out | 104 ++-- .../clientpositive/perf/spark/query72.q.out | 461 + .../clientpositive/perf/spark/query73.q.out | 34 +- .../clientpositive/perf/spark/query75.q.out | 248 +- .../clientpositive/perf/spark/query76.q.out | 130 +++-- .../clientpositive/perf/spark/query77.q.out | 66 +-- .../clientpositive/perf/spark/query78.q.out | 273 +-- .../clientpositive/perf/spark/query79.q.out | 10 +- .../clientpositive/perf/spark/query8.q.out | 174 +++ .../clientpositive/perf/spark/query80.q.out | 417 .../clientpositive/perf/spark/query81.q.out | 270 +- .../clientpositive/perf/spark/query82.q.out | 16 +- .../clientpositive/perf/spark/query83.q.out | 354 +++--- .../clientpositive/perf/spark/query85.q.out | 309 ++-- .../clientpositive/perf/spark/query88.q.out | 272 +- .../clientpositive/perf/spark/query89.q.out | 26 +- .../clientpositive/perf/spark/query90.q.out | 68 +-- .../clientpositive/perf/spark/query91.q.out | 14 +- .../clientpositive/perf/spark/query92.q.out | 106 ++-- .../clientpositive/perf/spark/query94.q.out | 70 +-- .../clientpositive/perf/spark/query95.q.out | 100 ++-- .../clientpositive/perf/spark/query97.q.out | 54 +- .../clientpositive/perf/spark/query99.q.out | 42 +- 63 files changed, 4603 insertions(+), 4536 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/data/conf/perf-reg/spark/hive-site.xml -- diff --git
[04/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query78.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out b/ql/src/test/results/clientpositive/perf/spark/query78.q.out index c2e8577..1467c5f 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out @@ -120,41 +120,57 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 164), Map 14 (PARTITION-LEVEL SORT, 164) -Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 85), Reducer 11 (PARTITION-LEVEL SORT, 85) -Reducer 13 <- Reducer 12 (GROUP, 93) -Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) +Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 14 (PARTITION-LEVEL SORT, 85) +Reducer 12 <- Reducer 11 (GROUP, 93) +Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 164), Map 15 (PARTITION-LEVEL SORT, 164) +Reducer 17 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 20 (PARTITION-LEVEL SORT, 85) Reducer 18 <- Reducer 17 (GROUP, 93) -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL SORT, 432) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 219), Reducer 8 (PARTITION-LEVEL SORT, 219) Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 177), Map 21 (PARTITION-LEVEL SORT, 177) -Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 219), Reducer 2 (PARTITION-LEVEL SORT, 219) -Reducer 4 <- Reducer 3 (GROUP, 241) -Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 167), Reducer 4 (PARTITION-LEVEL SORT, 167) -Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 5 (PARTITION-LEVEL SORT, 91) -Reducer 7 <- Reducer 6 (SORT, 1) +Reducer 3 <- Reducer 2 (GROUP, 241) +Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 167), Reducer 3 (PARTITION-LEVEL SORT, 167) +Reducer 5 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 4 (PARTITION-LEVEL SORT, 91) +Reducer 6 <- Reducer 5 (SORT, 1) +Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 432), Map 9 (PARTITION-LEVEL SORT, 432) A masked pattern was here Vertices: Map 1 Map Operator Tree: TableScan - alias: store_sales - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE + alias: date_dim + Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE Filter Operator -predicate: ss_sold_date_sk is not null (type: boolean) -Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE +predicate: ((d_year = 2000) and d_date_sk is not null) (type: boolean) +Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price (type: decimal(7,2)) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 - Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE + expressions: d_date_sk (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator -key expressions: _col1 (type: int), _col3 (type: int) -sort order: ++ -Map-reduce partition columns: _col1 (type: int), _col3 (type: int) -Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col2 (type: int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2)) +key expressions: _col0 (type: int) +sort order: + +Map-reduce partition columns: _col0 (type: int) +Statistics: Num rows: 36524 Data size: 40870356 Basic stats: COMPLETE Column stats: NONE Map 10 Map Operator Tree: TableScan + alias: date_dim + Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column stats: NONE + Filter Operator +
[01/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master b3fe6522e -> 244ca8e5c http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query92.q.out -- diff --git a/ql/src/test/results/clientpositive/perf/spark/query92.q.out b/ql/src/test/results/clientpositive/perf/spark/query92.q.out index 70c4c5a..e7b8632 100644 --- a/ql/src/test/results/clientpositive/perf/spark/query92.q.out +++ b/ql/src/test/results/clientpositive/perf/spark/query92.q.out @@ -67,7 +67,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 7 +Map 5 Map Operator Tree: TableScan alias: date_dim @@ -90,7 +90,7 @@ STAGE PLANS: Spark A masked pattern was here Vertices: -Map 10 +Map 9 Map Operator Tree: TableScan alias: date_dim @@ -112,11 +112,11 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: -Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 156), Map 6 (PARTITION-LEVEL SORT, 156) -Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 270), Reducer 9 (PARTITION-LEVEL SORT, 270) -Reducer 4 <- Reducer 3 (GROUP, 1) -Reducer 5 <- Reducer 4 (SORT, 1) -Reducer 9 <- Map 8 (GROUP, 169) +Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 261), Reducer 8 (PARTITION-LEVEL SORT, 261) +Reducer 3 <- Reducer 2 (GROUP, 1) +Reducer 4 <- Reducer 3 (SORT, 1) +Reducer 7 <- Map 6 (GROUP, 169) +Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 87), Reducer 7 (PARTITION-LEVEL SORT, 87) A masked pattern was here Vertices: Map 1 @@ -131,13 +131,25 @@ STAGE PLANS: expressions: ws_sold_date_sk (type: int), ws_item_sk (type: int), ws_ext_discount_amt (type: decimal(7,2)) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator -key expressions: _col1 (type: int) -sort order: + -Map-reduce partition columns: _col1 (type: int) -Statistics: Num rows: 144002668 Data size: 19580198212 Basic stats: COMPLETE Column stats: NONE -value expressions: _col0 (type: int), _col2 (type: decimal(7,2)) -Map 6 + Map Join Operator +condition map: + Inner Join 0 to 1 +keys: + 0 _col0 (type: int) + 1 _col0 (type: int) +outputColumnNames: _col1, _col2 +input vertices: + 1 Map 5 +Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: decimal(7,2)) +Local Work: + Map Reduce Local Work +Map 10 Map Operator Tree: TableScan alias: item @@ -154,7 +166,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE Column stats: NONE -Map 8 +Map 6 Map Operator Tree: TableScan alias: web_sales @@ -174,7 +186,7 @@ STAGE PLANS: 1 _col0 (type: int) outputColumnNames: _col1, _col2 input vertices: - 1 Map 10 + 1 Map 9 Statistics: Num rows: 158402938 Data size: 21538218500 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: sum(_col2), count(_col2) @@ -191,50 +203,22 @@ STAGE PLANS: Local Work: Map Reduce Local Work Reducer 2 -Local Work: - Map Reduce Local Work Reduce Operator Tree: Join Operator condition map: Inner Join 0 to 1 keys: 0 _col1 (type: int) - 1 _col0 (type: int) -
hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)
Repository: hive Updated Branches: refs/heads/branch-3 f57c33607 -> a3d9c46df HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3d9c46d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3d9c46d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3d9c46d Branch: refs/heads/branch-3 Commit: a3d9c46df453f787c3eaeeb2c33373997d17dec4 Parents: f57c336 Author: Vaibhav GumashtaAuthored: Wed Apr 11 15:39:30 2018 -0700 Committer: Vaibhav Gumashta Committed: Wed Apr 11 15:46:26 2018 -0700 -- .../hive/metastore/cache/CachedStore.java | 198 +++ 1 file changed, 114 insertions(+), 84 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/a3d9c46d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java -- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index c47856d..1ce86bb 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -18,23 +18,21 @@ package org.apache.hadoop.hive.metastore.cache; -import java.io.Closeable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; +import java.util.EmptyStackException; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Stack; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -100,7 +98,6 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SchemaVersion; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; @@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable { // Time after which metastore cache is updated from metastore DB by the background update thread private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD; private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false); + private static TablesPendingPrewarm tblsPendingPrewarm = new TablesPendingPrewarm(); private RawStore rawStore = null; private Configuration conf; private PartitionExpressionProxy expressionProxy = null; @@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable { static final private Logger LOG = LoggerFactory.getLogger(CachedStore.class.getName()); - public CachedStore() { - - } - @Override public void setConf(Configuration conf) { setConfInternal(conf); @@ -211,12 +205,13 @@ public class CachedStore implements RawStore, Configurable { Collection catalogsToCache; try { catalogsToCache = catalogsToCache(rawStore); -LOG.info("Going to cache catalogs: " + -org.apache.commons.lang.StringUtils.join(catalogsToCache, ", ")); +LOG.info("Going to cache catalogs: " ++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", ")); List catalogs = new ArrayList<>(catalogsToCache.size()); -for (String catName : catalogsToCache) catalogs.add(rawStore.getCatalog(catName)); +for (String catName : catalogsToCache) + catalogs.add(rawStore.getCatalog(catName)); sharedCache.populateCatalogsInCache(catalogs); - } catch (MetaException|NoSuchObjectException e) { + } catch (MetaException | NoSuchObjectException e) { LOG.warn("Failed to populate catalogs in cache, going to try again", e); // try again continue; @@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable { databases.add(rawStore.getDatabase(catName, dbName)); } catch
hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)
Repository: hive Updated Branches: refs/heads/master 42187fdbc -> b3fe6522e HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b3fe6522 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b3fe6522 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b3fe6522 Branch: refs/heads/master Commit: b3fe6522e651fa4f00f1a1a75e6f12c132eacf21 Parents: 42187fd Author: Vaibhav GumashtaAuthored: Wed Apr 11 15:39:30 2018 -0700 Committer: Vaibhav Gumashta Committed: Wed Apr 11 15:39:30 2018 -0700 -- .../hive/metastore/cache/CachedStore.java | 198 +++ 1 file changed, 114 insertions(+), 84 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/b3fe6522/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java -- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index c47856d..1ce86bb 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -18,23 +18,21 @@ package org.apache.hadoop.hive.metastore.cache; -import java.io.Closeable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; +import java.util.EmptyStackException; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Stack; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -100,7 +98,6 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SchemaVersion; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; @@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable { // Time after which metastore cache is updated from metastore DB by the background update thread private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD; private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false); + private static TablesPendingPrewarm tblsPendingPrewarm = new TablesPendingPrewarm(); private RawStore rawStore = null; private Configuration conf; private PartitionExpressionProxy expressionProxy = null; @@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable { static final private Logger LOG = LoggerFactory.getLogger(CachedStore.class.getName()); - public CachedStore() { - - } - @Override public void setConf(Configuration conf) { setConfInternal(conf); @@ -211,12 +205,13 @@ public class CachedStore implements RawStore, Configurable { Collection catalogsToCache; try { catalogsToCache = catalogsToCache(rawStore); -LOG.info("Going to cache catalogs: " + -org.apache.commons.lang.StringUtils.join(catalogsToCache, ", ")); +LOG.info("Going to cache catalogs: " ++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", ")); List catalogs = new ArrayList<>(catalogsToCache.size()); -for (String catName : catalogsToCache) catalogs.add(rawStore.getCatalog(catName)); +for (String catName : catalogsToCache) + catalogs.add(rawStore.getCatalog(catName)); sharedCache.populateCatalogsInCache(catalogs); - } catch (MetaException|NoSuchObjectException e) { + } catch (MetaException | NoSuchObjectException e) { LOG.warn("Failed to populate catalogs in cache, going to try again", e); // try again continue; @@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable { databases.add(rawStore.getDatabase(catName, dbName)); } catch
hive git commit: HIVE-18883: Add findbugs to yetus pre-commit checks (Sahil Takiar, reviewed by Peter Vary)
Repository: hive Updated Branches: refs/heads/master 8777125b3 -> 42187fdbc HIVE-18883: Add findbugs to yetus pre-commit checks (Sahil Takiar, reviewed by Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42187fdb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42187fdb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42187fdb Branch: refs/heads/master Commit: 42187fdbc3de0be793319483e2f463b3e8756f20 Parents: 8777125 Author: Sahil TakiarAuthored: Wed Apr 11 11:01:11 2018 -0700 Committer: Sahil Takiar Committed: Wed Apr 11 12:08:24 2018 -0700 -- dev-support/yetus-wrapper.sh| 47 .../findbugs/findbugs-exclude.xml | 24 + standalone-metastore/pom.xml| 57 +++- 3 files changed, 117 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/42187fdb/dev-support/yetus-wrapper.sh -- diff --git a/dev-support/yetus-wrapper.sh b/dev-support/yetus-wrapper.sh index 3a814d5..58da1d2 100755 --- a/dev-support/yetus-wrapper.sh +++ b/dev-support/yetus-wrapper.sh @@ -91,6 +91,34 @@ if [[ $? != 0 ]]; then fi HIVE_PATCHPROCESS=${mytmpdir} +CURLBIN=$(command -v curl) + +# Set FindBugs Home +FINDBUGS_VERSION="3.0.1" +if [[ ! -d "${HIVE_PATCHPROCESS}/findbugs-${FINDBUGS_VERSION}/" ]]; then + # Download FindBugs + FINDBUGS_BASEURL="http://prdownloads.sourceforge.net/findbugs/; + FINDBUGS_TARBALL="findbugs-${FINDBUGS_VERSION}.tar" + + pushd "${HIVE_PATCHPROCESS}" >/dev/null + if [[ -n "${CURLBIN}" ]]; then +"${CURLBIN}" -f -s -L -O "${FINDBUGS_BASEURL}/${FINDBUGS_TARBALL}.gz" +if [[ $? != 0 ]]; then + yetus_error "ERROR: yetus-dl: unable to download ${FINDBUGS_BASEURL}/${FINDBUGS_TARBALL}.gz" + exit 1 +fi + fi + + gunzip -c "${FINDBUGS_TARBALL}.gz" | tar xpf - + if [[ $? != 0 ]]; then +yetus_error "ERROR: ${FINDBUGS_TARBALL}.gz is corrupt. Investigate and then remove ${HIVE_PATCHPROCESS} to try again." +exit 1 + fi + popd >/dev/null +fi + +export FINDBUGS_HOME=${HIVE_PATCHPROCESS}/findbugs-${FINDBUGS_VERSION} + ## ## if we've already DL'd it, then short cut ## @@ -102,11 +130,10 @@ fi ## need to DL, etc ## -BASEURL="https://archive.apache.org/dist/yetus/${HIVE_YETUS_VERSION}/; -TARBALL="yetus-${HIVE_YETUS_VERSION}-bin.tar" +YETUS_BASEURL="https://archive.apache.org/dist/yetus/${HIVE_YETUS_VERSION}/; +YETUS_TARBALL="yetus-${HIVE_YETUS_VERSION}-bin.tar" GPGBIN=$(command -v gpg) -CURLBIN=$(command -v curl) pushd "${HIVE_PATCHPROCESS}" >/dev/null if [[ $? != 0 ]]; then @@ -115,9 +142,9 @@ if [[ $? != 0 ]]; then fi if [[ -n "${CURLBIN}" ]]; then - "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz" + "${CURLBIN}" -f -s -L -O "${YETUS_BASEURL}/${YETUS_TARBALL}.gz" if [[ $? != 0 ]]; then -yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz" +yetus_error "ERROR: yetus-dl: unable to download ${YETUS_BASEURL}/${YETUS_TARBALL}.gz" exit 1 fi else @@ -141,9 +168,9 @@ if [[ -n "${GPGBIN}" ]]; then yetus_error "ERROR: yetus-dl: unable to fetch https://dist.apache.org/repos/dist/release/yetus/KEYS; exit 1 fi - "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc" + "${CURLBIN}" -s -L -O "${YETUS_BASEURL}/${YETUS_TARBALL}.gz.asc" if [[ $? != 0 ]]; then -yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc" +yetus_error "ERROR: yetus-dl: unable to fetch ${YETUS_BASEURL}/${YETUS_TARBALL}.gz.asc" exit 1 fi "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --import "${HIVE_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1 @@ -151,16 +178,16 @@ if [[ -n "${GPGBIN}" ]]; then yetus_error "ERROR: yetus-dl: gpg unable to import ${HIVE_PATCHPROCESS}/KEYS_YETUS" exit 1 fi - "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --verify "${TARBALL}.gz.asc" >/dev/null 2>&1 + "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --verify "${YETUS_TARBALL}.gz.asc" >/dev/null 2>&1 if [[ $? != 0 ]]; then yetus_error "ERROR: yetus-dl: gpg verify of tarball in ${HIVE_PATCHPROCESS} failed" exit 1 fi fi -gunzip -c "${TARBALL}.gz" | tar xpf - +gunzip -c "${YETUS_TARBALL}.gz" | tar xpf - if [[ $? != 0 ]]; then - yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HIVE_PATCHPROCESS} to try again." + yetus_error "ERROR: ${YETUS_TARBALL}.gz is corrupt. Investigate and then remove ${HIVE_PATCHPROCESS} to try again." exit 1 fi http://git-wip-us.apache.org/repos/asf/hive/blob/42187fdb/standalone-metastore/findbugs/findbugs-exclude.xml
hive git commit: HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg, reviewed by Zoltan Haindrich)
Repository: hive Updated Branches: refs/heads/branch-3 0821efcad -> f57c33607 HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg,reviewed by Zoltan Haindrich) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f57c3360 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f57c3360 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f57c3360 Branch: refs/heads/branch-3 Commit: f57c336072ec5ae9af1b5b624eb4ec516ccdad0e Parents: 0821efc Author: Vineet GargAuthored: Wed Apr 11 11:17:36 2018 -0700 Committer: Vineet Garg Committed: Wed Apr 11 11:21:51 2018 -0700 -- ql/src/test/queries/clientpositive/sysdb.q | 5 +- .../results/clientpositive/llap/sysdb.q.out | 140 ++- 2 files changed, 12 insertions(+), 133 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f57c3360/ql/src/test/queries/clientpositive/sysdb.q -- diff --git a/ql/src/test/queries/clientpositive/sysdb.q b/ql/src/test/queries/clientpositive/sysdb.q index 36d80e2..399c3ce 100644 --- a/ql/src/test/queries/clientpositive/sysdb.q +++ b/ql/src/test/queries/clientpositive/sysdb.q @@ -3,6 +3,7 @@ set hive.strict.checks.cartesian.product=false; set hive.compute.query.using.stats=false; set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.cbo.enable=false; @@ -42,10 +43,6 @@ select grantor, principal_name from db_privs order by grantor, principal_name li select grantor, principal_name from global_privs order by grantor, principal_name limit 5; -select index_name, index_handler_class from idxs order by index_name limit 5; - -select param_key, param_value from index_params order by param_key, param_value limit 5; - select part_name from partitions order by part_name limit 5; select pkey_name, pkey_type from partition_keys order by pkey_name limit 5; http://git-wip-us.apache.org/repos/asf/hive/blob/f57c3360/ql/src/test/results/clientpositive/llap/sysdb.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index 73c5077..7d977d3 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -482,110 +482,6 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@GLOBAL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@IDXS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: SYS@IDXS -POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, - `PARAM_VALUE` string, - CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"PARAM_KEY\", - \"PARAM_VALUE\" -FROM - \"INDEX_PARAMS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@INDEX_PARAMS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, -
hive git commit: HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg, reviewed by Zoltan Haindrich)
Repository: hive Updated Branches: refs/heads/master f8a671d8c -> 8777125b3 HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg,reviewed by Zoltan Haindrich) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8777125b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8777125b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8777125b Branch: refs/heads/master Commit: 8777125b3d9dfcea7b7ce31f6535c5e64780bfbd Parents: f8a671d Author: Vineet GargAuthored: Wed Apr 11 11:17:36 2018 -0700 Committer: Vineet Garg Committed: Wed Apr 11 11:18:06 2018 -0700 -- ql/src/test/queries/clientpositive/sysdb.q | 5 +- .../results/clientpositive/llap/sysdb.q.out | 140 ++- 2 files changed, 12 insertions(+), 133 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/8777125b/ql/src/test/queries/clientpositive/sysdb.q -- diff --git a/ql/src/test/queries/clientpositive/sysdb.q b/ql/src/test/queries/clientpositive/sysdb.q index 36d80e2..399c3ce 100644 --- a/ql/src/test/queries/clientpositive/sysdb.q +++ b/ql/src/test/queries/clientpositive/sysdb.q @@ -3,6 +3,7 @@ set hive.strict.checks.cartesian.product=false; set hive.compute.query.using.stats=false; set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.cbo.enable=false; @@ -42,10 +43,6 @@ select grantor, principal_name from db_privs order by grantor, principal_name li select grantor, principal_name from global_privs order by grantor, principal_name limit 5; -select index_name, index_handler_class from idxs order by index_name limit 5; - -select param_key, param_value from index_params order by param_key, param_value limit 5; - select part_name from partitions order by part_name limit 5; select pkey_name, pkey_type from partition_keys order by pkey_name limit 5; http://git-wip-us.apache.org/repos/asf/hive/blob/8777125b/ql/src/test/results/clientpositive/llap/sysdb.q.out -- diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index 73c5077..7d977d3 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -482,110 +482,6 @@ FROM POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@GLOBAL_PRIVS POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@IDXS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` ( - `INDEX_ID` bigint, - `CREATE_TIME` int, - `DEFERRED_REBUILD` boolean, - `INDEX_HANDLER_CLASS` string, - `INDEX_NAME` string, - `INDEX_TBL_ID` bigint, - `LAST_ACCESS_TIME` int, - `ORIG_TBL_ID` bigint, - `SD_ID` bigint, - CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"CREATE_TIME\", - \"DEFERRED_REBUILD\", - \"INDEX_HANDLER_CLASS\", - \"INDEX_NAME\", - \"INDEX_TBL_ID\", - \"LAST_ACCESS_TIME\", - \"ORIG_TBL_ID\", - \"SD_ID\" -FROM - \"IDXS\"" -) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: SYS@IDXS -POSTHOOK: Output: database:sys -PREHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, - `PARAM_VALUE` string, - CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE -) -STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' -TBLPROPERTIES ( -"hive.sql.database.type" = "METASTORE", -"hive.sql.query" = -"SELECT - \"INDEX_ID\", - \"PARAM_KEY\", - \"PARAM_VALUE\" -FROM - \"INDEX_PARAMS\"" -) -PREHOOK: type: CREATETABLE -PREHOOK: Output: SYS@INDEX_PARAMS -PREHOOK: Output: database:sys -POSTHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` ( - `INDEX_ID` bigint, - `PARAM_KEY` string, -
hive git commit: HIVE-19077: Handle duplicate ptests requests standing in queue at the same time - addendum
Repository: hive Updated Branches: refs/heads/master 7c1f01004 -> f8a671d8c HIVE-19077: Handle duplicate ptests requests standing in queue at the same time - addendum Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8a671d8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8a671d8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8a671d8 Branch: refs/heads/master Commit: f8a671d8cfe8a26d1d12c51f93207ec92577c796 Parents: 7c1f010 Author: Adam SzitaAuthored: Wed Apr 11 10:49:24 2018 +0200 Committer: Peter Vary Committed: Wed Apr 11 10:49:24 2018 +0200 -- .../hive/ptest/api/client/JenkinsQueueUtil.java | 18 ++ 1 file changed, 14 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/f8a671d8/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java -- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java index f973781..f335164 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java @@ -20,6 +20,8 @@ package org.apache.hive.ptest.api.client; import java.io.IOException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; @@ -27,7 +29,9 @@ import org.apache.commons.cli.CommandLine; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import com.fasterxml.jackson.databind.JsonNode; @@ -50,7 +54,6 @@ public class JenkinsQueueUtil { private static final String ISSUE_FIELD_KEY = "ISSUE_NUM"; private static final String JIRA_KEY_PREFIX = "HIVE-"; - /** * Looks up the current queue of the precommit job on a jenkins instance (specified by * PTestClient.JENKINS_QUEUE_URL), and checks if current Jira is standing in queue already (i.e. @@ -112,11 +115,15 @@ public class JenkinsQueueUtil { private static String httpGet(String url) throws IOException { -DefaultHttpClient httpClient = new DefaultHttpClient(); + HttpGet request = new HttpGet(url); try { + CloseableHttpClient httpClient = HttpClientBuilder + .create() + .setSslcontext(SSLContexts.custom().useProtocol("TLSv1.2").build()) + .setRetryHandler(new PTestClient.PTestHttpRequestRetryHandler()) + .build(); request.addHeader("content-type", "application/json"); - httpClient.setHttpRequestRetryHandler(new PTestClient.PTestHttpRequestRetryHandler()); HttpResponse httpResponse = httpClient.execute(request); StatusLine statusLine = httpResponse.getStatusLine(); if (statusLine.getStatusCode() != 200) { @@ -124,6 +131,9 @@ public class JenkinsQueueUtil { } String response = EntityUtils.toString(httpResponse.getEntity(), "UTF-8"); return response; +} catch (NoSuchAlgorithmException | KeyManagementException e) { + e.printStackTrace(); + throw new IOException(e.getMessage()); } finally { request.abort(); }
hive git commit: HIVE-19077: Handle duplicate ptests requests standing in queue at the same time (Adam Szita, via Peter Vary)
Repository: hive Updated Branches: refs/heads/master f6dbcac35 -> 7c1f01004 HIVE-19077: Handle duplicate ptests requests standing in queue at the same time (Adam Szita, via Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c1f0100 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c1f0100 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c1f0100 Branch: refs/heads/master Commit: 7c1f010045db3af6fa5b805ae4f49fc9b07d5639 Parents: f6dbcac Author: Adam SzitaAuthored: Wed Apr 11 09:45:56 2018 +0200 Committer: Peter Vary Committed: Wed Apr 11 09:45:56 2018 +0200 -- dev-support/jenkins-common.sh | 2 + dev-support/jenkins-execute-build.sh| 3 +- .../hive/ptest/api/client/JenkinsQueueUtil.java | 133 +++ .../hive/ptest/api/client/PTestClient.java | 15 ++- 4 files changed, 150 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-common.sh -- diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh index 0467d11..64f486f 100644 --- a/dev-support/jenkins-common.sh +++ b/dev-support/jenkins-common.sh @@ -15,6 +15,8 @@ # limitations under the License. JIRA_ROOT_URL="https://issues.apache.org; +JENKINS_URL="https://builds.apache.org; +JENKINS_QUEUE_QUERY="/queue/api/json?tree=items[task[name],inQueueSince,actions[parameters[name,value]],why]" fail() { echo "$@" 1>&2 http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-execute-build.sh -- diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh index f660fcb..35392dd 100644 --- a/dev-support/jenkins-execute-build.sh +++ b/dev-support/jenkins-execute-build.sh @@ -51,7 +51,8 @@ call_ptest_server() { local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-3.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*" java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \ - --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@" + --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" \ + --jenkinsQueueUrl "$JENKINS_URL$JENKINS_QUEUE_QUERY" "$@" } # Unpack all test results http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java -- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java new file mode 100644 index 000..f973781 --- /dev/null +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hive.ptest.api.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; + +/** + * Utility class for the Precommit test job queue on Jenkins + */ +public class JenkinsQueueUtil { + + private static final String JSON_ITEMS_FIELD = "items"; + private static final String JSON_TASK_FIELD = "task"; + private static final String JSON_TASK_NAME_FIELD = "name"; +