hive git commit: HIVE-19130: NPE is thrown when REPL LOAD applied drop partition event (Sankar Hariappan, reviewed by Mahesh Kumar Behera, Thejas M Nair)

[sankarh]

Repository: hive
Updated Branches:
  refs/heads/master 244ca8e5c -> d9fae0493


HIVE-19130: NPE is thrown when REPL LOAD applied drop partition event (Sankar 
Hariappan, reviewed by Mahesh Kumar Behera, Thejas M Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d9fae049
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d9fae049
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d9fae049

Branch: refs/heads/master
Commit: d9fae049305e20ec8a72e581a2fc938028523402
Parents: 244ca8e
Author: Sankar Hariappan 
Authored: Thu Apr 12 10:29:47 2018 +0530
Committer: Sankar Hariappan 
Committed: Thu Apr 12 10:29:47 2018 +0530

--
 ...TestReplicationScenariosAcrossInstances.java | 55 
 .../hadoop/hive/ql/parse/WarehouseInstance.java |  5 ++
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  8 ++-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   | 10 
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 31 +--
 .../parse/repl/load/message/TableHandler.java   |  1 +
 6 files changed, 106 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/d9fae049/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
--
diff --git 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 689ca76..70e1aa7 100644
--- 
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ 
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -620,4 +620,59 @@ public class TestReplicationScenariosAcrossInstances {
 .run("show functions like '" + replicatedDbName + "*'")
 .verifyResult(null);
   }
+
+  @Test
+  public void testIncrementalReplWithEventsBatchHavingDropCreateTable() throws 
Throwable {
+// Bootstrap dump with empty db
+WarehouseInstance.Tuple bootstrapTuple = primary.dump(primaryDbName, null);
+
+// Bootstrap load in replica
+replica.load(replicatedDbName, bootstrapTuple.dumpLocation)
+.status(replicatedDbName)
+.verifyResult(bootstrapTuple.lastReplicationId);
+
+// First incremental dump
+WarehouseInstance.Tuple firstIncremental = primary.run("use " + 
primaryDbName)
+.run("create table table1 (i int)")
+.run("create table table2 (id int) partitioned by (country 
string)")
+.run("insert into table1 values (1)")
+.run("insert into table2 partition(country='india') values(1)")
+.dump(primaryDbName, bootstrapTuple.lastReplicationId);
+
+// Second incremental dump
+WarehouseInstance.Tuple secondIncremental = primary.run("use " + 
primaryDbName)
+.run("drop table table1")
+.run("drop table table2")
+.run("create table table2 (id int) partitioned by (country 
string)")
+.run("alter table table2 add partition(country='india')")
+.run("alter table table2 drop partition(country='india')")
+.run("insert into table2 partition(country='us') values(2)")
+.run("create table table1 (i int)")
+.run("insert into table1 values (2)")
+.dump(primaryDbName, firstIncremental.lastReplicationId);
+
+// First incremental load
+replica.load(replicatedDbName, firstIncremental.dumpLocation)
+.status(replicatedDbName)
+.verifyResult(firstIncremental.lastReplicationId)
+.run("use " + replicatedDbName)
+.run("show tables")
+.verifyResults(new String[] {"table1", "table2"})
+.run("select * from table1")
+.verifyResults(new String[] {"1"})
+.run("select id from table2 order by id")
+.verifyResults(new String[] {"1"});
+
+// Second incremental load
+replica.load(replicatedDbName, secondIncremental.dumpLocation)
+.status(replicatedDbName)
+.verifyResult(secondIncremental.lastReplicationId)
+.run("use " + replicatedDbName)
+.run("show tables")
+.verifyResults(new String[] {"table1", "table2"})
+.run("select * from table1")
+.verifyResults(new String[] {"2"})
+.run("select id from table2 order by id")
+.verifyResults(new String[] {"2"});
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/d9fae049/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java

[11/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query29.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query29.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query29.q.out
index 456fce7..530a625 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query29.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query29.q.out
@@ -124,7 +124,7 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 14 
+Map 13 
 Map Operator Tree:
 TableScan
   alias: d2
@@ -146,51 +146,51 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 10 <- Map 12 (PARTITION-LEVEL SORT, 306), Map 9 
(PARTITION-LEVEL SORT, 306)
-Reducer 11 <- Map 13 (PARTITION-LEVEL SORT, 374), Reducer 10 
(PARTITION-LEVEL SORT, 374)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 398), Map 7 (PARTITION-LEVEL 
SORT, 398)
-Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 442), Reducer 2 
(PARTITION-LEVEL SORT, 442)
-Reducer 4 <- Reducer 11 (PARTITION-LEVEL SORT, 850), Reducer 3 
(PARTITION-LEVEL SORT, 850)
-Reducer 5 <- Reducer 4 (GROUP, 582)
-Reducer 6 <- Reducer 5 (SORT, 1)
+Reducer 10 <- Map 14 (PARTITION-LEVEL SORT, 486), Reducer 9 
(PARTITION-LEVEL SORT, 486)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 6 (PARTITION-LEVEL 
SORT, 306)
+Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 917), Reducer 2 
(PARTITION-LEVEL SORT, 917)
+Reducer 4 <- Reducer 3 (GROUP, 640)
+Reducer 5 <- Reducer 4 (SORT, 1)
+Reducer 8 <- Map 11 (PARTITION-LEVEL SORT, 398), Map 7 
(PARTITION-LEVEL SORT, 398)
+Reducer 9 <- Map 12 (PARTITION-LEVEL SORT, 476), Reducer 8 
(PARTITION-LEVEL SORT, 476)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: store_sales
-  Statistics: Num rows: 575995635 Data size: 50814502088 Basic 
stats: COMPLETE Column stats: NONE
+  alias: catalog_sales
+  Statistics: Num rows: 287989836 Data size: 38999608952 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
-predicate: (ss_customer_sk is not null and ss_item_sk is 
not null and ss_sold_date_sk is not null and ss_store_sk is not null and 
ss_ticket_number is not null) (type: boolean)
-Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
+predicate: (cs_bill_customer_sk is not null and cs_item_sk 
is not null and cs_sold_date_sk is not null) (type: boolean)
+Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: ss_sold_date_sk (type: int), ss_item_sk 
(type: int), ss_customer_sk (type: int), ss_store_sk (type: int), 
ss_ticket_number (type: int), ss_quantity (type: int)
-  outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
-  Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
+  expressions: cs_sold_date_sk (type: int), 
cs_bill_customer_sk (type: int), cs_item_sk (type: int), cs_quantity (type: int)
+  outputColumnNames: _col0, _col1, _col2, _col3
+  Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
-Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col1 (type: int), _col2 (type: 
int), _col3 (type: int), _col4 (type: int), _col5 (type: int)
-Map 12 
+Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
+value expressions: _col1 (type: int), _col2 (type: 
int), _col3 (type: int)
+Map 11 
 Map Operator Tree:
 TableScan
-  alias: d3
+  alias: d1
   Statistics: Num rows: 73049 Data size: 81741831 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
-predicate: ((d_year) IN (1999, 2000, 2001) and d_date_sk 
is not null) (type: boolean)
-Statistics: Num rows: 36525 Data size: 40871475 Basic 
stats: COMPLETE Column stats: NONE
+predicate: 

[09/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query45.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query45.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query45.q.out
index 7e1cc88..b674400 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query45.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query45.q.out
@@ -46,10 +46,10 @@ STAGE PLANS:
   Stage: Stage-2
 Spark
   Edges:
-Reducer 6 <- Map 5 (GROUP, 1)
+Reducer 16 <- Map 15 (GROUP, 1)
  A masked pattern was here 
   Vertices:
-Map 5 
+Map 15 
 Map Operator Tree:
 TableScan
   alias: item
@@ -70,7 +70,7 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 1 Data size: 16 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: bigint), _col1 
(type: bigint)
-Reducer 6 
+Reducer 16 
 Local Work:
   Map Reduce Local Work
 Reduce Operator Tree:
@@ -88,45 +88,33 @@ STAGE PLANS:
 Spark
   Edges:
 Reducer 11 <- Map 10 (GROUP, 3)
-Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 15 
(PARTITION-LEVEL SORT, 154)
-Reducer 14 <- Map 16 (PARTITION-LEVEL SORT, 706), Reducer 13 
(PARTITION-LEVEL SORT, 706)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 527), Reducer 9 
(PARTITION-LEVEL SORT, 527)
-Reducer 3 <- Reducer 2 (GROUP, 224)
-Reducer 4 <- Reducer 3 (SORT, 1)
+Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 154), Map 14 
(PARTITION-LEVEL SORT, 154)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 855), Map 6 (PARTITION-LEVEL 
SORT, 855)
+Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 777), Reducer 9 
(PARTITION-LEVEL SORT, 777)
+Reducer 4 <- Reducer 3 (GROUP, 230)
+Reducer 5 <- Reducer 4 (SORT, 1)
 Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 7), Reducer 11 
(PARTITION-LEVEL SORT, 7)
-Reducer 9 <- Reducer 14 (PARTITION-LEVEL SORT, 191), Reducer 8 
(PARTITION-LEVEL SORT, 191)
+Reducer 9 <- Reducer 13 (PARTITION-LEVEL SORT, 174), Reducer 8 
(PARTITION-LEVEL SORT, 174)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: customer_address
-  Statistics: Num rows: 4000 Data size: 40595195284 Basic 
stats: COMPLETE Column stats: NONE
+  alias: customer
+  Statistics: Num rows: 8000 Data size: 68801615852 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
-predicate: ca_address_sk is not null (type: boolean)
-Statistics: Num rows: 4000 Data size: 40595195284 
Basic stats: COMPLETE Column stats: NONE
+predicate: (c_current_addr_sk is not null and 
c_customer_sk is not null) (type: boolean)
+Statistics: Num rows: 8000 Data size: 68801615852 
Basic stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: ca_address_sk (type: int), ca_county (type: 
string), ca_zip (type: string)
-  outputColumnNames: _col0, _col1, _col2
-  Statistics: Num rows: 4000 Data size: 40595195284 
Basic stats: COMPLETE Column stats: NONE
-  Map Join Operator
-condition map:
- Inner Join 0 to 1
-keys:
-  0 
-  1 
-outputColumnNames: _col0, _col1, _col2, _col3, _col4
-input vertices:
-  1 Reducer 6
-Statistics: Num rows: 4000 Data size: 41275195284 
Basic stats: COMPLETE Column stats: NONE
-Reduce Output Operator
-  key expressions: _col0 (type: int)
-  sort order: +
-  Map-reduce partition columns: _col0 (type: int)
-  Statistics: Num rows: 4000 Data size: 
41275195284 Basic stats: COMPLETE Column stats: NONE
-  value expressions: _col1 (type: string), _col2 
(type: string), _col3 (type: bigint), _col4 (type: bigint)
-Local Work:
-  Map Reduce Local Work
+  expressions: c_customer_sk (type: int), 
c_current_addr_sk (type: int)
+  outputColumnNames: _col0, _col1
+  Statistics: Num rows: 8000 Data size: 68801615852 
Basic stats: COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col1 

[05/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query75.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query75.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query75.q.out
index f4fca34..4b76570 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query75.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query75.q.out
@@ -144,25 +144,25 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 400), Map 16 
(PARTITION-LEVEL SORT, 400)
-Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 438), Reducer 13 
(PARTITION-LEVEL SORT, 438)
+Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 398), Map 16 
(PARTITION-LEVEL SORT, 398)
+Reducer 14 <- Map 10 (PARTITION-LEVEL SORT, 440), Reducer 13 
(PARTITION-LEVEL SORT, 440)
 Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 516), Reducer 14 
(PARTITION-LEVEL SORT, 516)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 308), Map 16 
(PARTITION-LEVEL SORT, 308)
-Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 156), Map 19 
(PARTITION-LEVEL SORT, 156)
-Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 169), Reducer 20 
(PARTITION-LEVEL SORT, 169)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 16 
(PARTITION-LEVEL SORT, 306)
+Reducer 20 <- Map 16 (PARTITION-LEVEL SORT, 154), Map 19 
(PARTITION-LEVEL SORT, 154)
+Reducer 21 <- Map 10 (PARTITION-LEVEL SORT, 171), Reducer 20 
(PARTITION-LEVEL SORT, 171)
 Reducer 22 <- Map 25 (PARTITION-LEVEL SORT, 196), Reducer 21 
(PARTITION-LEVEL SORT, 196)
-Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 308), Map 32 
(PARTITION-LEVEL SORT, 308)
-Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 336), Reducer 27 
(PARTITION-LEVEL SORT, 336)
+Reducer 27 <- Map 26 (PARTITION-LEVEL SORT, 306), Map 32 
(PARTITION-LEVEL SORT, 306)
+Reducer 28 <- Map 33 (PARTITION-LEVEL SORT, 338), Reducer 27 
(PARTITION-LEVEL SORT, 338)
 Reducer 29 <- Map 34 (PARTITION-LEVEL SORT, 393), Reducer 28 
(PARTITION-LEVEL SORT, 393)
-Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 336), Reducer 2 
(PARTITION-LEVEL SORT, 336)
+Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 338), Reducer 2 
(PARTITION-LEVEL SORT, 338)
 Reducer 30 <- Reducer 29 (GROUP, 934), Reducer 38 (GROUP, 934)
 Reducer 31 <- Reducer 30 (GROUP PARTITION-LEVEL SORT, 671), Reducer 45 
(GROUP PARTITION-LEVEL SORT, 671)
-Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 400), Map 35 
(PARTITION-LEVEL SORT, 400)
-Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 438), Reducer 36 
(PARTITION-LEVEL SORT, 438)
+Reducer 36 <- Map 32 (PARTITION-LEVEL SORT, 398), Map 35 
(PARTITION-LEVEL SORT, 398)
+Reducer 37 <- Map 33 (PARTITION-LEVEL SORT, 440), Reducer 36 
(PARTITION-LEVEL SORT, 440)
 Reducer 38 <- Map 41 (PARTITION-LEVEL SORT, 516), Reducer 37 
(PARTITION-LEVEL SORT, 516)
 Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 393), Reducer 3 
(PARTITION-LEVEL SORT, 393)
-Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 156), Map 42 
(PARTITION-LEVEL SORT, 156)
-Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 169), Reducer 43 
(PARTITION-LEVEL SORT, 169)
+Reducer 43 <- Map 32 (PARTITION-LEVEL SORT, 154), Map 42 
(PARTITION-LEVEL SORT, 154)
+Reducer 44 <- Map 33 (PARTITION-LEVEL SORT, 171), Reducer 43 
(PARTITION-LEVEL SORT, 171)
 Reducer 45 <- Map 48 (PARTITION-LEVEL SORT, 196), Reducer 44 
(PARTITION-LEVEL SORT, 196)
 Reducer 5 <- Reducer 15 (GROUP, 934), Reducer 4 (GROUP, 934)
 Reducer 6 <- Reducer 22 (GROUP PARTITION-LEVEL SORT, 671), Reducer 5 
(GROUP PARTITION-LEVEL SORT, 671)
@@ -183,28 +183,29 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1, _col2, _col3, _col4
   Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
   Reduce Output Operator
-key expressions: _col1 (type: int)
+key expressions: _col0 (type: int)
 sort order: +
-Map-reduce partition columns: _col1 (type: int)
+Map-reduce partition columns: _col0 (type: int)
 Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col0 (type: int), _col2 (type: 
int), _col3 (type: int), _col4 (type: decimal(7,2))
+value expressions: _col1 (type: int), _col2 (type: 
int), _col3 (type: int), _col4 (type: decimal(7,2))
 Map 10 
 Map Operator Tree:
 TableScan
-  alias: date_dim
-  Statistics: Num rows: 73049 Data size: 81741831 Basic 

[10/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query4.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query4.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query4.q.out
index 5074d49..75c4fbb 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query4.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query4.q.out
@@ -225,20 +225,20 @@ STAGE PLANS:
 Reducer 10 <- Map 13 (PARTITION-LEVEL SORT, 398), Map 9 
(PARTITION-LEVEL SORT, 398)
 Reducer 11 <- Map 14 (PARTITION-LEVEL SORT, 975), Reducer 10 
(PARTITION-LEVEL SORT, 975)
 Reducer 12 <- Reducer 11 (GROUP, 481)
-Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 306), Map 19 
(PARTITION-LEVEL SORT, 306)
-Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 873), Reducer 16 
(PARTITION-LEVEL SORT, 873)
-Reducer 18 <- Reducer 17 (GROUP, 369)
+Reducer 16 <- Map 15 (PARTITION-LEVEL SORT, 398), Map 19 
(PARTITION-LEVEL SORT, 398)
+Reducer 17 <- Map 20 (PARTITION-LEVEL SORT, 975), Reducer 16 
(PARTITION-LEVEL SORT, 975)
+Reducer 18 <- Reducer 17 (GROUP, 481)
 Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 306), Map 7 (PARTITION-LEVEL 
SORT, 306)
-Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 154), Map 25 
(PARTITION-LEVEL SORT, 154)
-Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 706), Reducer 22 
(PARTITION-LEVEL SORT, 706)
-Reducer 24 <- Reducer 23 (GROUP, 186)
+Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 306), Map 25 
(PARTITION-LEVEL SORT, 306)
+Reducer 23 <- Map 26 (PARTITION-LEVEL SORT, 873), Reducer 22 
(PARTITION-LEVEL SORT, 873)
+Reducer 24 <- Reducer 23 (GROUP, 369)
 Reducer 28 <- Map 27 (PARTITION-LEVEL SORT, 154), Map 31 
(PARTITION-LEVEL SORT, 154)
 Reducer 29 <- Map 32 (PARTITION-LEVEL SORT, 706), Reducer 28 
(PARTITION-LEVEL SORT, 706)
 Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 873), Reducer 2 
(PARTITION-LEVEL SORT, 873)
 Reducer 30 <- Reducer 29 (GROUP, 186)
-Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 398), Map 37 
(PARTITION-LEVEL SORT, 398)
-Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 975), Reducer 34 
(PARTITION-LEVEL SORT, 975)
-Reducer 36 <- Reducer 35 (GROUP, 481)
+Reducer 34 <- Map 33 (PARTITION-LEVEL SORT, 154), Map 37 
(PARTITION-LEVEL SORT, 154)
+Reducer 35 <- Map 38 (PARTITION-LEVEL SORT, 706), Reducer 34 
(PARTITION-LEVEL SORT, 706)
+Reducer 36 <- Reducer 35 (GROUP, 186)
 Reducer 4 <- Reducer 3 (GROUP, 369)
 Reducer 5 <- Reducer 12 (PARTITION-LEVEL SORT, 690), Reducer 18 
(PARTITION-LEVEL SORT, 690), Reducer 24 (PARTITION-LEVEL SORT, 690), Reducer 30 
(PARTITION-LEVEL SORT, 690), Reducer 36 (PARTITION-LEVEL SORT, 690), Reducer 4 
(PARTITION-LEVEL SORT, 690)
 Reducer 6 <- Reducer 5 (SORT, 1)
@@ -300,20 +300,20 @@ STAGE PLANS:
 Map 15 
 Map Operator Tree:
 TableScan
-  alias: catalog_sales
-  Statistics: Num rows: 287989836 Data size: 38999608952 Basic 
stats: COMPLETE Column stats: NONE
+  alias: store_sales
+  Statistics: Num rows: 575995635 Data size: 50814502088 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
-predicate: (cs_bill_customer_sk is not null and 
cs_sold_date_sk is not null) (type: boolean)
-Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
+predicate: (ss_customer_sk is not null and ss_sold_date_sk 
is not null) (type: boolean)
+Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: cs_sold_date_sk (type: int), 
cs_bill_customer_sk (type: int), cs_ext_discount_amt (type: decimal(7,2)), 
cs_ext_sales_price (type: decimal(7,2)), cs_ext_wholesale_cost (type: 
decimal(7,2)), cs_ext_list_price (type: decimal(7,2))
+  expressions: ss_sold_date_sk (type: int), ss_customer_sk 
(type: int), ss_ext_discount_amt (type: decimal(7,2)), ss_ext_sales_price 
(type: decimal(7,2)), ss_ext_wholesale_cost (type: decimal(7,2)), 
ss_ext_list_price (type: decimal(7,2))
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
-  Statistics: Num rows: 287989836 Data size: 38999608952 
Basic stats: COMPLETE Column stats: NONE
+  Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
   Reduce Output Operator
 key expressions: _col0 (type: int)
 sort order: +
 Map-reduce partition columns: _col0 

[03/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query81.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query81.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query81.q.out
index 23998d6..5b2d5b3 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query81.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query81.q.out
@@ -66,37 +66,72 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 25), Map 14 
(PARTITION-LEVEL SORT, 25)
-Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 344), Reducer 11 
(PARTITION-LEVEL SORT, 344)
-Reducer 13 <- Reducer 12 (GROUP PARTITION-LEVEL SORT, 349)
-Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 697), Map 18 
(PARTITION-LEVEL SORT, 697)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 25), Map 8 (PARTITION-LEVEL 
SORT, 25)
-Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 344), Reducer 2 
(PARTITION-LEVEL SORT, 344)
-Reducer 4 <- Reducer 3 (GROUP, 349)
-Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 262), Reducer 4 
(PARTITION-LEVEL SORT, 262)
-Reducer 6 <- Reducer 17 (PARTITION-LEVEL SORT, 656), Reducer 5 
(PARTITION-LEVEL SORT, 656)
-Reducer 7 <- Reducer 6 (SORT, 1)
+Reducer 10 <- Reducer 16 (PARTITION-LEVEL SORT, 262), Reducer 9 
(PARTITION-LEVEL SORT, 262)
+Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 25), Map 17 
(PARTITION-LEVEL SORT, 25)
+Reducer 15 <- Map 18 (PARTITION-LEVEL SORT, 344), Reducer 14 
(PARTITION-LEVEL SORT, 344)
+Reducer 16 <- Reducer 15 (GROUP PARTITION-LEVEL SORT, 349)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 697), Map 5 (PARTITION-LEVEL 
SORT, 697)
+Reducer 3 <- Reducer 10 (PARTITION-LEVEL SORT, 656), Reducer 2 
(PARTITION-LEVEL SORT, 656)
+Reducer 4 <- Reducer 3 (SORT, 1)
+Reducer 7 <- Map 11 (PARTITION-LEVEL SORT, 25), Map 6 (PARTITION-LEVEL 
SORT, 25)
+Reducer 8 <- Map 12 (PARTITION-LEVEL SORT, 344), Reducer 7 
(PARTITION-LEVEL SORT, 344)
+Reducer 9 <- Reducer 8 (GROUP, 349)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: catalog_returns
-  Statistics: Num rows: 28798881 Data size: 3057234680 Basic 
stats: COMPLETE Column stats: NONE
+  alias: customer
+  Statistics: Num rows: 8000 Data size: 68801615852 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
-predicate: (cr_returned_date_sk is not null and 
cr_returning_addr_sk is not null and cr_returning_customer_sk is not null) 
(type: boolean)
-Statistics: Num rows: 28798881 Data size: 3057234680 Basic 
stats: COMPLETE Column stats: NONE
+predicate: (c_current_addr_sk is not null and 
c_customer_sk is not null) (type: boolean)
+Statistics: Num rows: 8000 Data size: 68801615852 
Basic stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: cr_returned_date_sk (type: int), 
cr_returning_customer_sk (type: int), cr_returning_addr_sk (type: int), 
cr_return_amt_inc_tax (type: decimal(7,2))
-  outputColumnNames: _col0, _col1, _col2, _col3
-  Statistics: Num rows: 28798881 Data size: 3057234680 
Basic stats: COMPLETE Column stats: NONE
+  expressions: c_customer_sk (type: int), c_customer_id 
(type: string), c_current_addr_sk (type: int), c_salutation (type: string), 
c_first_name (type: string), c_last_name (type: string)
+  outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5
+  Statistics: Num rows: 8000 Data size: 68801615852 
Basic stats: COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col2 (type: int)
+sort order: +
+Map-reduce partition columns: _col2 (type: int)
+Statistics: Num rows: 8000 Data size: 68801615852 
Basic stats: COMPLETE Column stats: NONE
+value expressions: _col0 (type: int), _col1 (type: 
string), _col3 (type: string), _col4 (type: string), _col5 (type: string)
+Map 11 
+Map Operator Tree:
+TableScan
+  alias: date_dim
+  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: 
COMPLETE Column stats: NONE
+  Filter Operator
+predicate: ((d_year = 1998) and d_date_sk is not null) 
(type: boolean)
+Statistics: Num rows: 36524 Data size: 40870356 Basic 
stats: COMPLETE Column stats: NONE
+

[07/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query58.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query58.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query58.q.out
index eb5dffe..f06cbef 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query58.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query58.q.out
@@ -1,6 +1,6 @@
-Warning: Map Join MAPJOIN[180][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
-Warning: Map Join MAPJOIN[181][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
-Warning: Map Join MAPJOIN[182][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
+Warning: Map Join MAPJOIN[183][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
+Warning: Map Join MAPJOIN[184][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
+Warning: Map Join MAPJOIN[185][bigTable=?] in task 'Stage-1:MAPRED' is a cross 
product
 PREHOOK: query: explain
 with ss_items as
  (select i_item_id item_id
@@ -140,10 +140,10 @@ STAGE PLANS:
   Stage: Stage-2
 Spark
   Edges:
-Reducer 12 <- Map 11 (GROUP, 1)
+Reducer 11 <- Map 10 (GROUP, 1)
  A masked pattern was here 
   Vertices:
-Map 11 
+Map 10 
 Map Operator Tree:
 TableScan
   alias: date_dim
@@ -162,7 +162,7 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: bigint)
-Reducer 12 
+Reducer 11 
 Local Work:
   Map Reduce Local Work
 Reduce Operator Tree:
@@ -184,10 +184,10 @@ STAGE PLANS:
   Stage: Stage-3
 Spark
   Edges:
-Reducer 25 <- Map 24 (GROUP, 1)
+Reducer 24 <- Map 23 (GROUP, 1)
  A masked pattern was here 
   Vertices:
-Map 24 
+Map 23 
 Map Operator Tree:
 TableScan
   alias: date_dim
@@ -206,7 +206,7 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: bigint)
-Reducer 25 
+Reducer 24 
 Local Work:
   Map Reduce Local Work
 Reduce Operator Tree:
@@ -228,10 +228,10 @@ STAGE PLANS:
   Stage: Stage-4
 Spark
   Edges:
-Reducer 38 <- Map 37 (GROUP, 1)
+Reducer 37 <- Map 36 (GROUP, 1)
  A masked pattern was here 
   Vertices:
-Map 37 
+Map 36 
 Map Operator Tree:
 TableScan
   alias: date_dim
@@ -250,7 +250,7 @@ STAGE PLANS:
   sort order: 
   Statistics: Num rows: 1 Data size: 8 Basic stats: 
COMPLETE Column stats: NONE
   value expressions: _col0 (type: bigint)
-Reducer 38 
+Reducer 37 
 Local Work:
   Map Reduce Local Work
 Reduce Operator Tree:
@@ -272,47 +272,76 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 10 <- Reducer 9 (GROUP, 2)
-Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 2), Reducer 23 
(PARTITION-LEVEL SORT, 2)
-Reducer 18 <- Map 27 (PARTITION-LEVEL SORT, 398), Reducer 17 
(PARTITION-LEVEL SORT, 398)
-Reducer 19 <- Map 28 (PARTITION-LEVEL SORT, 442), Reducer 18 
(PARTITION-LEVEL SORT, 442)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2), Reducer 10 
(PARTITION-LEVEL SORT, 2)
-Reducer 20 <- Reducer 19 (GROUP, 481)
-Reducer 22 <- Map 21 (PARTITION-LEVEL SORT, 2), Map 26 
(PARTITION-LEVEL SORT, 2)
-Reducer 23 <- Reducer 22 (GROUP, 2)
-Reducer 3 <- Map 14 (PARTITION-LEVEL SORT, 306), Reducer 2 
(PARTITION-LEVEL SORT, 306)
-Reducer 30 <- Map 29 (PARTITION-LEVEL SORT, 2), Reducer 36 
(PARTITION-LEVEL SORT, 2)
-Reducer 31 <- Map 40 (PARTITION-LEVEL SORT, 154), Reducer 30 
(PARTITION-LEVEL SORT, 154)
-Reducer 32 <- Map 41 (PARTITION-LEVEL SORT, 174), Reducer 31 
(PARTITION-LEVEL SORT, 174)
-Reducer 33 <- Reducer 32 (GROUP, 186)
-Reducer 35 <- Map 34 (PARTITION-LEVEL SORT, 2), Map 39 
(PARTITION-LEVEL SORT, 2)
-Reducer 36 <- Reducer 35 (GROUP, 2)
-Reducer 4 <- Map 15 (PARTITION-LEVEL SORT, 341), Reducer 3 
(PARTITION-LEVEL SORT, 341)
-Reducer 5 <- Reducer 4 (GROUP, 369)
-Reducer 6 <- Reducer 20 (PARTITION-LEVEL SORT, 518), Reducer 33 
(PARTITION-LEVEL SORT, 518), Reducer 5 (PARTITION-LEVEL SORT, 518)
-Reducer 7 <- Reducer 6 (SORT, 1)
-Reducer 9 <- Map 13 (PARTITION-LEVEL SORT, 2), Map 8 (PARTITION-LEVEL 
SORT, 2)
+Reducer 13 <- Map 12 

[06/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query65.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query65.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query65.q.out
index 575fc5c..3b3baef 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query65.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query65.q.out
@@ -57,8 +57,7 @@ POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-2 is a root stage
   Stage-3 depends on stages: Stage-2
-  Stage-4 depends on stages: Stage-3
-  Stage-1 depends on stages: Stage-4
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
@@ -66,29 +65,6 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 11 
-Map Operator Tree:
-TableScan
-  alias: store
-  Statistics: Num rows: 1704 Data size: 3256276 Basic stats: 
COMPLETE Column stats: NONE
-  Filter Operator
-predicate: s_store_sk is not null (type: boolean)
-Statistics: Num rows: 1704 Data size: 3256276 Basic stats: 
COMPLETE Column stats: NONE
-Select Operator
-  expressions: s_store_sk (type: int), s_store_name (type: 
string)
-  outputColumnNames: _col0, _col1
-  Statistics: Num rows: 1704 Data size: 3256276 Basic 
stats: COMPLETE Column stats: NONE
-  Spark HashTable Sink Operator
-keys:
-  0 _col0 (type: int)
-  1 _col0 (type: int)
-Local Work:
-  Map Reduce Local Work
-
-  Stage: Stage-3
-Spark
- A masked pattern was here 
-  Vertices:
 Map 6 
 Map Operator Tree:
 TableScan
@@ -108,7 +84,7 @@ STAGE PLANS:
 Local Work:
   Map Reduce Local Work
 
-  Stage: Stage-4
+  Stage: Stage-3
 Spark
  A masked pattern was here 
   Vertices:
@@ -135,8 +111,8 @@ STAGE PLANS:
 Spark
   Edges:
 Reducer 2 <- Map 1 (GROUP, 437)
-Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 328), Reducer 8 
(PARTITION-LEVEL SORT, 328)
-Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 86), Reducer 3 
(PARTITION-LEVEL SORT, 86)
+Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 328), Reducer 2 
(PARTITION-LEVEL SORT, 328), Reducer 8 (PARTITION-LEVEL SORT, 328)
+Reducer 4 <- Map 11 (PARTITION-LEVEL SORT, 166), Reducer 3 
(PARTITION-LEVEL SORT, 166)
 Reducer 5 <- Reducer 4 (SORT, 1)
 Reducer 8 <- Map 7 (GROUP PARTITION-LEVEL SORT, 437)
  A masked pattern was here 
@@ -180,6 +156,24 @@ STAGE PLANS:
 Map 10 
 Map Operator Tree:
 TableScan
+  alias: store
+  Statistics: Num rows: 1704 Data size: 3256276 Basic stats: 
COMPLETE Column stats: NONE
+  Filter Operator
+predicate: s_store_sk is not null (type: boolean)
+Statistics: Num rows: 1704 Data size: 3256276 Basic stats: 
COMPLETE Column stats: NONE
+Select Operator
+  expressions: s_store_sk (type: int), s_store_name (type: 
string)
+  outputColumnNames: _col0, _col1
+  Statistics: Num rows: 1704 Data size: 3256276 Basic 
stats: COMPLETE Column stats: NONE
+  Reduce Output Operator
+key expressions: _col0 (type: int)
+sort order: +
+Map-reduce partition columns: _col0 (type: int)
+Statistics: Num rows: 1704 Data size: 3256276 Basic 
stats: COMPLETE Column stats: NONE
+value expressions: _col1 (type: string)
+Map 11 
+Map Operator Tree:
+TableScan
   alias: item
   Statistics: Num rows: 462000 Data size: 663560457 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
@@ -250,23 +244,23 @@ STAGE PLANS:
   Join Operator
 condition map:
  Inner Join 0 to 1
+ Inner Join 0 to 2
 keys:
   0 _col0 (type: int)
   1 _col0 (type: int)
-outputColumnNames: _col0, _col1, _col2, _col4
-Statistics: Num rows: 348477374 Data size: 30742775095 Basic 
stats: COMPLETE Column stats: NONE
+  2 _col0 (type: int)
+outputColumnNames: _col1, _col2, _col4, _col6
+Statistics: Num rows: 696954748 Data size: 61485550191 Basic 
stats: COMPLETE Column stats: NONE
 Filter Operator
   

[08/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query50.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query50.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query50.q.out
index b978201..fde0c4e 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query50.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query50.q.out
@@ -124,7 +124,7 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 6 
+Map 10 
 Map Operator Tree:
 TableScan
   alias: store
@@ -138,7 +138,7 @@ STAGE PLANS:
   Statistics: Num rows: 1704 Data size: 3256276 Basic 
stats: COMPLETE Column stats: NONE
   Spark HashTable Sink Operator
 keys:
-  0 _col3 (type: int)
+  0 _col10 (type: int)
   1 _col0 (type: int)
 Local Work:
   Map Reduce Local Work
@@ -146,63 +146,16 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 476), Reducer 8 
(PARTITION-LEVEL SORT, 476)
-Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 481), Reducer 2 
(PARTITION-LEVEL SORT, 481)
-Reducer 4 <- Reducer 3 (GROUP, 529)
-Reducer 5 <- Reducer 4 (SORT, 1)
-Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 36), Map 9 (PARTITION-LEVEL 
SORT, 36)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 36), Map 7 (PARTITION-LEVEL 
SORT, 36)
+Reducer 3 <- Map 8 (PARTITION-LEVEL SORT, 436), Reducer 2 
(PARTITION-LEVEL SORT, 436)
+Reducer 4 <- Map 9 (PARTITION-LEVEL SORT, 438), Reducer 3 
(PARTITION-LEVEL SORT, 438)
+Reducer 5 <- Reducer 4 (GROUP, 529)
+Reducer 6 <- Reducer 5 (SORT, 1)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: store_sales
-  Statistics: Num rows: 575995635 Data size: 50814502088 Basic 
stats: COMPLETE Column stats: NONE
-  Filter Operator
-predicate: (ss_customer_sk is not null and ss_item_sk is 
not null and ss_sold_date_sk is not null and ss_store_sk is not null and 
ss_ticket_number is not null) (type: boolean)
-Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
-Select Operator
-  expressions: ss_sold_date_sk (type: int), ss_item_sk 
(type: int), ss_customer_sk (type: int), ss_store_sk (type: int), 
ss_ticket_number (type: int)
-  outputColumnNames: _col0, _col1, _col2, _col3, _col4
-  Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
-  Map Join Operator
-condition map:
- Inner Join 0 to 1
-keys:
-  0 _col3 (type: int)
-  1 _col0 (type: int)
-outputColumnNames: _col0, _col1, _col2, _col4, _col6, 
_col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
-input vertices:
-  1 Map 6
-Statistics: Num rows: 633595212 Data size: 55895953508 
Basic stats: COMPLETE Column stats: NONE
-Reduce Output Operator
-  key expressions: _col1 (type: int), _col2 (type: 
int), _col4 (type: int)
-  sort order: +++
-  Map-reduce partition columns: _col1 (type: int), 
_col2 (type: int), _col4 (type: int)
-  Statistics: Num rows: 633595212 Data size: 
55895953508 Basic stats: COMPLETE Column stats: NONE
-  value expressions: _col0 (type: int), _col6 (type: 
string), _col7 (type: int), _col8 (type: string), _col9 (type: string), _col10 
(type: string), _col11 (type: string), _col12 (type: string), _col13 (type: 
string), _col14 (type: string), _col15 (type: string)
-Local Work:
-  Map Reduce Local Work
-Map 10 
-Map Operator Tree:
-TableScan
-  alias: d1
-  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: 
COMPLETE Column stats: NONE
-  Filter Operator
-predicate: d_date_sk is not null (type: boolean)
-Statistics: Num rows: 73049 Data size: 81741831 Basic 
stats: COMPLETE Column stats: NONE
-Select Operator
-  expressions: d_date_sk (type: int)
-  outputColumnNames: _col0
-  Statistics: Num rows: 

[02/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query85.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query85.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
index c82dcf7..d60751c 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query85.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query85.q.out
@@ -175,7 +175,7 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 15 
+Map 13 
 Map Operator Tree:
 TableScan
   alias: reason
@@ -212,21 +212,21 @@ STAGE PLANS:
   Statistics: Num rows: 4602 Data size: 2696178 Basic 
stats: COMPLETE Column stats: NONE
   Spark HashTable Sink Operator
 keys:
-  0 _col0 (type: int)
-  1 _col2 (type: int)
+  0 _col10 (type: int)
+  1 _col0 (type: int)
 Local Work:
   Map Reduce Local Work
 
   Stage: Stage-1
 Spark
   Edges:
-Reducer 13 <- Map 12 (PARTITION-LEVEL SORT, 20), Map 14 
(PARTITION-LEVEL SORT, 20)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 90), Map 8 (PARTITION-LEVEL 
SORT, 90)
-Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 17), Reducer 2 
(PARTITION-LEVEL SORT, 17)
-Reducer 4 <- Map 10 (PARTITION-LEVEL SORT, 19), Reducer 3 
(PARTITION-LEVEL SORT, 19)
-Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 35), Reducer 4 
(PARTITION-LEVEL SORT, 35)
-Reducer 6 <- Reducer 5 (GROUP, 2)
-Reducer 7 <- Reducer 6 (SORT, 1)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 28), Map 9 (PARTITION-LEVEL 
SORT, 28)
+Reducer 3 <- Map 10 (PARTITION-LEVEL SORT, 98), Reducer 2 
(PARTITION-LEVEL SORT, 98)
+Reducer 4 <- Map 12 (PARTITION-LEVEL SORT, 5), Reducer 3 
(PARTITION-LEVEL SORT, 5)
+Reducer 5 <- Map 14 (PARTITION-LEVEL SORT, 11), Reducer 4 
(PARTITION-LEVEL SORT, 11)
+Reducer 6 <- Map 15 (PARTITION-LEVEL SORT, 7), Reducer 5 
(PARTITION-LEVEL SORT, 7)
+Reducer 7 <- Reducer 6 (GROUP, 7)
+Reducer 8 <- Reducer 7 (SORT, 1)
  A masked pattern was here 
   Vertices:
 Map 1 
@@ -242,61 +242,32 @@ STAGE PLANS:
   outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6, _col7
   Statistics: Num rows: 14398467 Data size: 1325194184 
Basic stats: COMPLETE Column stats: NONE
   Reduce Output Operator
-key expressions: _col2 (type: int)
-sort order: +
-Map-reduce partition columns: _col2 (type: int)
+key expressions: _col0 (type: int), _col5 (type: int)
+sort order: ++
+Map-reduce partition columns: _col0 (type: int), _col5 
(type: int)
 Statistics: Num rows: 14398467 Data size: 1325194184 
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col0 (type: int), _col1 (type: 
int), _col3 (type: int), _col4 (type: int), _col5 (type: int), _col6 (type: 
decimal(7,2)), _col7 (type: decimal(7,2))
+value expressions: _col1 (type: int), _col2 (type: 
int), _col3 (type: int), _col4 (type: int), _col6 (type: decimal(7,2)), _col7 
(type: decimal(7,2))
 Map 10 
 Map Operator Tree:
 TableScan
-  alias: cd2
-  Statistics: Num rows: 1861800 Data size: 717186159 Basic 
stats: COMPLETE Column stats: NONE
+  alias: customer_address
+  Statistics: Num rows: 4000 Data size: 40595195284 Basic 
stats: COMPLETE Column stats: NONE
   Filter Operator
-predicate: (((cd_education_status = '4 yr Degree') or 
(cd_education_status = 'Primary') or (cd_education_status = 'Advanced Degree')) 
and ((cd_marital_status = 'M') or (cd_marital_status = 'D') or 
(cd_marital_status = 'U')) and cd_demo_sk is not null and cd_education_status 
is not null and cd_marital_status is not null) (type: boolean)
-Statistics: Num rows: 1861800 Data size: 717186159 Basic 
stats: COMPLETE Column stats: NONE
+predicate: ((ca_country = 'United States') and (ca_state) 
IN ('KY', 'GA', 'NM', 'MT', 'OR', 'IN', 'WI', 'MO', 'WV') and ca_address_sk is 
not null) (type: boolean)
+Statistics: Num rows: 1000 Data size: 10148798821 
Basic stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: cd_demo_sk (type: int), cd_marital_status 
(type: string), cd_education_status (type: string)
-  outputColumnNames: _col0, 

[13/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan 
Haindrich via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan 


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/244ca8e5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/244ca8e5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/244ca8e5

Branch: refs/heads/master
Commit: 244ca8e5c3192acd017d691ccdbaf0fa06c9fe39
Parents: b3fe652
Author: Zoltan Haindrich 
Authored: Wed Apr 11 20:05:01 2018 -0700
Committer: Ashutosh Chauhan 
Committed: Wed Apr 11 20:05:01 2018 -0700

--
 data/conf/perf-reg/spark/hive-site.xml  |   6 +
 data/conf/perf-reg/tez/hive-site.xml|   5 +
 .../hive/cli/control/CorePerfCliDriver.java |   6 +-
 .../hadoop/hive/ql/MetaStoreDumpUtility.java| 231 +
 .../org/apache/hadoop/hive/ql/QTestUtil.java| 221 +
 .../hive/ql/parse/BaseSemanticAnalyzer.java |   2 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   2 +-
 .../clientpositive/perf/spark/query11.q.out |  48 +-
 .../clientpositive/perf/spark/query15.q.out | 164 +++
 .../clientpositive/perf/spark/query16.q.out |  70 +--
 .../clientpositive/perf/spark/query18.q.out | 216 
 .../clientpositive/perf/spark/query19.q.out | 218 -
 .../clientpositive/perf/spark/query21.q.out | 114 +++--
 .../clientpositive/perf/spark/query24.q.out | 282 +--
 .../clientpositive/perf/spark/query25.q.out | 118 ++---
 .../clientpositive/perf/spark/query29.q.out | 280 +--
 .../clientpositive/perf/spark/query30.q.out | 262 +-
 .../clientpositive/perf/spark/query32.q.out | 132 ++---
 .../clientpositive/perf/spark/query34.q.out |  34 +-
 .../clientpositive/perf/spark/query35.q.out |  74 ++-
 .../clientpositive/perf/spark/query37.q.out |  16 +-
 .../clientpositive/perf/spark/query4.q.out  | 214 
 .../clientpositive/perf/spark/query40.q.out | 116 +++--
 .../clientpositive/perf/spark/query44.q.out | 246 +-
 .../clientpositive/perf/spark/query45.q.out | 204 
 .../clientpositive/perf/spark/query46.q.out | 104 ++--
 .../clientpositive/perf/spark/query47.q.out |  92 ++--
 .../clientpositive/perf/spark/query48.q.out |  94 ++--
 .../clientpositive/perf/spark/query5.q.out  |  38 +-
 .../clientpositive/perf/spark/query50.q.out | 196 
 .../clientpositive/perf/spark/query53.q.out | 105 ++--
 .../clientpositive/perf/spark/query54.q.out | 277 ++-
 .../clientpositive/perf/spark/query57.q.out |  92 ++--
 .../clientpositive/perf/spark/query58.q.out | 490 +--
 .../clientpositive/perf/spark/query6.q.out  | 350 +++--
 .../clientpositive/perf/spark/query61.q.out |  40 +-
 .../clientpositive/perf/spark/query63.q.out | 105 ++--
 .../clientpositive/perf/spark/query65.q.out | 100 ++--
 .../clientpositive/perf/spark/query66.q.out |  20 +-
 .../clientpositive/perf/spark/query67.q.out | 137 +++---
 .../clientpositive/perf/spark/query68.q.out | 104 ++--
 .../clientpositive/perf/spark/query72.q.out | 461 +
 .../clientpositive/perf/spark/query73.q.out |  34 +-
 .../clientpositive/perf/spark/query75.q.out | 248 +-
 .../clientpositive/perf/spark/query76.q.out | 130 +++--
 .../clientpositive/perf/spark/query77.q.out |  66 +--
 .../clientpositive/perf/spark/query78.q.out | 273 +--
 .../clientpositive/perf/spark/query79.q.out |  10 +-
 .../clientpositive/perf/spark/query8.q.out  | 174 +++
 .../clientpositive/perf/spark/query80.q.out | 417 
 .../clientpositive/perf/spark/query81.q.out | 270 +-
 .../clientpositive/perf/spark/query82.q.out |  16 +-
 .../clientpositive/perf/spark/query83.q.out | 354 +++---
 .../clientpositive/perf/spark/query85.q.out | 309 ++--
 .../clientpositive/perf/spark/query88.q.out | 272 +-
 .../clientpositive/perf/spark/query89.q.out |  26 +-
 .../clientpositive/perf/spark/query90.q.out |  68 +--
 .../clientpositive/perf/spark/query91.q.out |  14 +-
 .../clientpositive/perf/spark/query92.q.out | 106 ++--
 .../clientpositive/perf/spark/query94.q.out |  70 +--
 .../clientpositive/perf/spark/query95.q.out | 100 ++--
 .../clientpositive/perf/spark/query97.q.out |  54 +-
 .../clientpositive/perf/spark/query99.q.out |  42 +-
 63 files changed, 4603 insertions(+), 4536 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/data/conf/perf-reg/spark/hive-site.xml
--
diff --git 

[04/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query78.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query78.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query78.q.out
index c2e8577..1467c5f 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query78.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query78.q.out
@@ -120,41 +120,57 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 164), Map 14 
(PARTITION-LEVEL SORT, 164)
-Reducer 12 <- Map 15 (PARTITION-LEVEL SORT, 85), Reducer 11 
(PARTITION-LEVEL SORT, 85)
-Reducer 13 <- Reducer 12 (GROUP, 93)
-Reducer 17 <- Map 16 (PARTITION-LEVEL SORT, 85), Reducer 20 
(PARTITION-LEVEL SORT, 85)
+Reducer 11 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 14 
(PARTITION-LEVEL SORT, 85)
+Reducer 12 <- Reducer 11 (GROUP, 93)
+Reducer 14 <- Map 13 (PARTITION-LEVEL SORT, 164), Map 15 
(PARTITION-LEVEL SORT, 164)
+Reducer 17 <- Map 10 (PARTITION-LEVEL SORT, 85), Reducer 20 
(PARTITION-LEVEL SORT, 85)
 Reducer 18 <- Reducer 17 (GROUP, 93)
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 432), Map 8 (PARTITION-LEVEL 
SORT, 432)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 219), Reducer 8 
(PARTITION-LEVEL SORT, 219)
 Reducer 20 <- Map 19 (PARTITION-LEVEL SORT, 177), Map 21 
(PARTITION-LEVEL SORT, 177)
-Reducer 3 <- Map 9 (PARTITION-LEVEL SORT, 219), Reducer 2 
(PARTITION-LEVEL SORT, 219)
-Reducer 4 <- Reducer 3 (GROUP, 241)
-Reducer 5 <- Reducer 13 (PARTITION-LEVEL SORT, 167), Reducer 4 
(PARTITION-LEVEL SORT, 167)
-Reducer 6 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 5 
(PARTITION-LEVEL SORT, 91)
-Reducer 7 <- Reducer 6 (SORT, 1)
+Reducer 3 <- Reducer 2 (GROUP, 241)
+Reducer 4 <- Reducer 12 (PARTITION-LEVEL SORT, 167), Reducer 3 
(PARTITION-LEVEL SORT, 167)
+Reducer 5 <- Reducer 18 (PARTITION-LEVEL SORT, 91), Reducer 4 
(PARTITION-LEVEL SORT, 91)
+Reducer 6 <- Reducer 5 (SORT, 1)
+Reducer 8 <- Map 7 (PARTITION-LEVEL SORT, 432), Map 9 (PARTITION-LEVEL 
SORT, 432)
  A masked pattern was here 
   Vertices:
 Map 1 
 Map Operator Tree:
 TableScan
-  alias: store_sales
-  Statistics: Num rows: 575995635 Data size: 50814502088 Basic 
stats: COMPLETE Column stats: NONE
+  alias: date_dim
+  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: 
COMPLETE Column stats: NONE
   Filter Operator
-predicate: ss_sold_date_sk is not null (type: boolean)
-Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
+predicate: ((d_year = 2000) and d_date_sk is not null) 
(type: boolean)
+Statistics: Num rows: 36524 Data size: 40870356 Basic 
stats: COMPLETE Column stats: NONE
 Select Operator
-  expressions: ss_sold_date_sk (type: int), ss_item_sk 
(type: int), ss_customer_sk (type: int), ss_ticket_number (type: int), 
ss_quantity (type: int), ss_wholesale_cost (type: decimal(7,2)), ss_sales_price 
(type: decimal(7,2))
-  outputColumnNames: _col0, _col1, _col2, _col3, _col4, 
_col5, _col6
-  Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
+  expressions: d_date_sk (type: int)
+  outputColumnNames: _col0
+  Statistics: Num rows: 36524 Data size: 40870356 Basic 
stats: COMPLETE Column stats: NONE
   Reduce Output Operator
-key expressions: _col1 (type: int), _col3 (type: int)
-sort order: ++
-Map-reduce partition columns: _col1 (type: int), _col3 
(type: int)
-Statistics: Num rows: 575995635 Data size: 50814502088 
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col0 (type: int), _col2 (type: 
int), _col4 (type: int), _col5 (type: decimal(7,2)), _col6 (type: decimal(7,2))
+key expressions: _col0 (type: int)
+sort order: +
+Map-reduce partition columns: _col0 (type: int)
+Statistics: Num rows: 36524 Data size: 40870356 Basic 
stats: COMPLETE Column stats: NONE
 Map 10 
 Map Operator Tree:
 TableScan
+  alias: date_dim
+  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: 
COMPLETE Column stats: NONE
+  Filter Operator
+   

[01/13] hive git commit: HIVE-19147 : Fix PerfCliDrivers: Tpcds30T missed CAT_NAME change (Zoltan Haindrich via Ashutosh Chauhan)

[hashutosh]

Repository: hive
Updated Branches:
  refs/heads/master b3fe6522e -> 244ca8e5c


http://git-wip-us.apache.org/repos/asf/hive/blob/244ca8e5/ql/src/test/results/clientpositive/perf/spark/query92.q.out
--
diff --git a/ql/src/test/results/clientpositive/perf/spark/query92.q.out 
b/ql/src/test/results/clientpositive/perf/spark/query92.q.out
index 70c4c5a..e7b8632 100644
--- a/ql/src/test/results/clientpositive/perf/spark/query92.q.out
+++ b/ql/src/test/results/clientpositive/perf/spark/query92.q.out
@@ -67,7 +67,7 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 7 
+Map 5 
 Map Operator Tree:
 TableScan
   alias: date_dim
@@ -90,7 +90,7 @@ STAGE PLANS:
 Spark
  A masked pattern was here 
   Vertices:
-Map 10 
+Map 9 
 Map Operator Tree:
 TableScan
   alias: date_dim
@@ -112,11 +112,11 @@ STAGE PLANS:
   Stage: Stage-1
 Spark
   Edges:
-Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 156), Map 6 (PARTITION-LEVEL 
SORT, 156)
-Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 270), Reducer 9 
(PARTITION-LEVEL SORT, 270)
-Reducer 4 <- Reducer 3 (GROUP, 1)
-Reducer 5 <- Reducer 4 (SORT, 1)
-Reducer 9 <- Map 8 (GROUP, 169)
+Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 261), Reducer 8 
(PARTITION-LEVEL SORT, 261)
+Reducer 3 <- Reducer 2 (GROUP, 1)
+Reducer 4 <- Reducer 3 (SORT, 1)
+Reducer 7 <- Map 6 (GROUP, 169)
+Reducer 8 <- Map 10 (PARTITION-LEVEL SORT, 87), Reducer 7 
(PARTITION-LEVEL SORT, 87)
  A masked pattern was here 
   Vertices:
 Map 1 
@@ -131,13 +131,25 @@ STAGE PLANS:
   expressions: ws_sold_date_sk (type: int), ws_item_sk 
(type: int), ws_ext_discount_amt (type: decimal(7,2))
   outputColumnNames: _col0, _col1, _col2
   Statistics: Num rows: 144002668 Data size: 19580198212 
Basic stats: COMPLETE Column stats: NONE
-  Reduce Output Operator
-key expressions: _col1 (type: int)
-sort order: +
-Map-reduce partition columns: _col1 (type: int)
-Statistics: Num rows: 144002668 Data size: 19580198212 
Basic stats: COMPLETE Column stats: NONE
-value expressions: _col0 (type: int), _col2 (type: 
decimal(7,2))
-Map 6 
+  Map Join Operator
+condition map:
+ Inner Join 0 to 1
+keys:
+  0 _col0 (type: int)
+  1 _col0 (type: int)
+outputColumnNames: _col1, _col2
+input vertices:
+  1 Map 5
+Statistics: Num rows: 158402938 Data size: 21538218500 
Basic stats: COMPLETE Column stats: NONE
+Reduce Output Operator
+  key expressions: _col1 (type: int)
+  sort order: +
+  Map-reduce partition columns: _col1 (type: int)
+  Statistics: Num rows: 158402938 Data size: 
21538218500 Basic stats: COMPLETE Column stats: NONE
+  value expressions: _col2 (type: decimal(7,2))
+Local Work:
+  Map Reduce Local Work
+Map 10 
 Map Operator Tree:
 TableScan
   alias: item
@@ -154,7 +166,7 @@ STAGE PLANS:
 sort order: +
 Map-reduce partition columns: _col0 (type: int)
 Statistics: Num rows: 231000 Data size: 331780228 
Basic stats: COMPLETE Column stats: NONE
-Map 8 
+Map 6 
 Map Operator Tree:
 TableScan
   alias: web_sales
@@ -174,7 +186,7 @@ STAGE PLANS:
   1 _col0 (type: int)
 outputColumnNames: _col1, _col2
 input vertices:
-  1 Map 10
+  1 Map 9
 Statistics: Num rows: 158402938 Data size: 21538218500 
Basic stats: COMPLETE Column stats: NONE
 Group By Operator
   aggregations: sum(_col2), count(_col2)
@@ -191,50 +203,22 @@ STAGE PLANS:
 Local Work:
   Map Reduce Local Work
 Reducer 2 
-Local Work:
-  Map Reduce Local Work
 Reduce Operator Tree:
   Join Operator
 condition map:
  Inner Join 0 to 1
 keys:
   0 _col1 (type: int)
-  1 _col0 (type: int)
-

hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)

[vgumashta]

Repository: hive
Updated Branches:
  refs/heads/branch-3 f57c33607 -> a3d9c46df


HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during 
prewarm (Vaibhav Gumashta reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3d9c46d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3d9c46d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3d9c46d

Branch: refs/heads/branch-3
Commit: a3d9c46df453f787c3eaeeb2c33373997d17dec4
Parents: f57c336
Author: Vaibhav Gumashta 
Authored: Wed Apr 11 15:39:30 2018 -0700
Committer: Vaibhav Gumashta 
Committed: Wed Apr 11 15:46:26 2018 -0700

--
 .../hive/metastore/cache/CachedStore.java   | 198 +++
 1 file changed, 114 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/a3d9c46d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index c47856d..1ce86bb 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -18,23 +18,21 @@
 package org.apache.hadoop.hive.metastore.cache;
 
 
-import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.EmptyStackException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -100,7 +98,6 @@ import 
org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
 import org.apache.hadoop.hive.metastore.api.SchemaVersion;
 import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.hadoop.hive.metastore.api.Type;
@@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable {
   // Time after which metastore cache is updated from metastore DB by the 
background update thread
   private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD;
   private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false);
+  private static TablesPendingPrewarm tblsPendingPrewarm = new 
TablesPendingPrewarm();
   private RawStore rawStore = null;
   private Configuration conf;
   private PartitionExpressionProxy expressionProxy = null;
@@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable 
{
 
   static final private Logger LOG = 
LoggerFactory.getLogger(CachedStore.class.getName());
 
-  public CachedStore() {
-
-  }
-
   @Override
   public void setConf(Configuration conf) {
 setConfInternal(conf);
@@ -211,12 +205,13 @@ public class CachedStore implements RawStore, 
Configurable {
   Collection catalogsToCache;
   try {
 catalogsToCache = catalogsToCache(rawStore);
-LOG.info("Going to cache catalogs: " +
-org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
+LOG.info("Going to cache catalogs: "
++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
 List catalogs = new ArrayList<>(catalogsToCache.size());
-for (String catName : catalogsToCache) 
catalogs.add(rawStore.getCatalog(catName));
+for (String catName : catalogsToCache)
+  catalogs.add(rawStore.getCatalog(catName));
 sharedCache.populateCatalogsInCache(catalogs);
-  } catch (MetaException|NoSuchObjectException e) {
+  } catch (MetaException | NoSuchObjectException e) {
 LOG.warn("Failed to populate catalogs in cache, going to try again", 
e);
 // try again
 continue;
@@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable {
   databases.add(rawStore.getDatabase(catName, dbName));
 } catch 

hive git commit: HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during prewarm (Vaibhav Gumashta reviewed by Daniel Dai)

[vgumashta]

Repository: hive
Updated Branches:
  refs/heads/master 42187fdbc -> b3fe6522e


HIVE-18840: CachedStore: Prioritize loading of recently accessed tables during 
prewarm (Vaibhav Gumashta reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b3fe6522
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b3fe6522
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b3fe6522

Branch: refs/heads/master
Commit: b3fe6522e651fa4f00f1a1a75e6f12c132eacf21
Parents: 42187fd
Author: Vaibhav Gumashta 
Authored: Wed Apr 11 15:39:30 2018 -0700
Committer: Vaibhav Gumashta 
Committed: Wed Apr 11 15:39:30 2018 -0700

--
 .../hive/metastore/cache/CachedStore.java   | 198 +++
 1 file changed, 114 insertions(+), 84 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/b3fe6522/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
--
diff --git 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index c47856d..1ce86bb 100644
--- 
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ 
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -18,23 +18,21 @@
 package org.apache.hadoop.hive.metastore.cache;
 
 
-import java.io.Closeable;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.EmptyStackException;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Stack;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -100,7 +98,6 @@ import 
org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
 import org.apache.hadoop.hive.metastore.api.SchemaVersion;
 import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.TableMeta;
 import org.apache.hadoop.hive.metastore.api.Type;
@@ -146,6 +143,7 @@ public class CachedStore implements RawStore, Configurable {
   // Time after which metastore cache is updated from metastore DB by the 
background update thread
   private static long cacheRefreshPeriodMS = DEFAULT_CACHE_REFRESH_PERIOD;
   private static AtomicBoolean isCachePrewarmed = new AtomicBoolean(false);
+  private static TablesPendingPrewarm tblsPendingPrewarm = new 
TablesPendingPrewarm();
   private RawStore rawStore = null;
   private Configuration conf;
   private PartitionExpressionProxy expressionProxy = null;
@@ -153,10 +151,6 @@ public class CachedStore implements RawStore, Configurable 
{
 
   static final private Logger LOG = 
LoggerFactory.getLogger(CachedStore.class.getName());
 
-  public CachedStore() {
-
-  }
-
   @Override
   public void setConf(Configuration conf) {
 setConfInternal(conf);
@@ -211,12 +205,13 @@ public class CachedStore implements RawStore, 
Configurable {
   Collection catalogsToCache;
   try {
 catalogsToCache = catalogsToCache(rawStore);
-LOG.info("Going to cache catalogs: " +
-org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
+LOG.info("Going to cache catalogs: "
++ org.apache.commons.lang.StringUtils.join(catalogsToCache, ", "));
 List catalogs = new ArrayList<>(catalogsToCache.size());
-for (String catName : catalogsToCache) 
catalogs.add(rawStore.getCatalog(catName));
+for (String catName : catalogsToCache)
+  catalogs.add(rawStore.getCatalog(catName));
 sharedCache.populateCatalogsInCache(catalogs);
-  } catch (MetaException|NoSuchObjectException e) {
+  } catch (MetaException | NoSuchObjectException e) {
 LOG.warn("Failed to populate catalogs in cache, going to try again", 
e);
 // try again
 continue;
@@ -232,8 +227,8 @@ public class CachedStore implements RawStore, Configurable {
   databases.add(rawStore.getDatabase(catName, dbName));
 } catch 

hive git commit: HIVE-18883: Add findbugs to yetus pre-commit checks (Sahil Takiar, reviewed by Peter Vary)

[stakiar]

Repository: hive
Updated Branches:
  refs/heads/master 8777125b3 -> 42187fdbc


HIVE-18883: Add findbugs to yetus pre-commit checks (Sahil Takiar, reviewed by 
Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42187fdb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42187fdb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42187fdb

Branch: refs/heads/master
Commit: 42187fdbc3de0be793319483e2f463b3e8756f20
Parents: 8777125
Author: Sahil Takiar 
Authored: Wed Apr 11 11:01:11 2018 -0700
Committer: Sahil Takiar 
Committed: Wed Apr 11 12:08:24 2018 -0700

--
 dev-support/yetus-wrapper.sh| 47 
 .../findbugs/findbugs-exclude.xml   | 24 +
 standalone-metastore/pom.xml| 57 +++-
 3 files changed, 117 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/42187fdb/dev-support/yetus-wrapper.sh
--
diff --git a/dev-support/yetus-wrapper.sh b/dev-support/yetus-wrapper.sh
index 3a814d5..58da1d2 100755
--- a/dev-support/yetus-wrapper.sh
+++ b/dev-support/yetus-wrapper.sh
@@ -91,6 +91,34 @@ if [[ $? != 0 ]]; then
 fi
 HIVE_PATCHPROCESS=${mytmpdir}
 
+CURLBIN=$(command -v curl)
+
+# Set FindBugs Home
+FINDBUGS_VERSION="3.0.1"
+if [[ ! -d "${HIVE_PATCHPROCESS}/findbugs-${FINDBUGS_VERSION}/" ]]; then
+  # Download FindBugs
+  FINDBUGS_BASEURL="http://prdownloads.sourceforge.net/findbugs/;
+  FINDBUGS_TARBALL="findbugs-${FINDBUGS_VERSION}.tar"
+
+  pushd "${HIVE_PATCHPROCESS}" >/dev/null
+  if [[ -n "${CURLBIN}" ]]; then
+"${CURLBIN}" -f -s -L -O "${FINDBUGS_BASEURL}/${FINDBUGS_TARBALL}.gz"
+if [[ $? != 0 ]]; then
+  yetus_error "ERROR: yetus-dl: unable to download 
${FINDBUGS_BASEURL}/${FINDBUGS_TARBALL}.gz"
+  exit 1
+fi
+  fi
+
+  gunzip -c "${FINDBUGS_TARBALL}.gz" | tar xpf -
+  if [[ $? != 0 ]]; then
+yetus_error "ERROR: ${FINDBUGS_TARBALL}.gz is corrupt. Investigate and 
then remove ${HIVE_PATCHPROCESS} to try again."
+exit 1
+  fi
+  popd >/dev/null
+fi
+
+export FINDBUGS_HOME=${HIVE_PATCHPROCESS}/findbugs-${FINDBUGS_VERSION}
+
 ##
 ## if we've already DL'd it, then short cut
 ##
@@ -102,11 +130,10 @@ fi
 ## need to DL, etc
 ##
 
-BASEURL="https://archive.apache.org/dist/yetus/${HIVE_YETUS_VERSION}/;
-TARBALL="yetus-${HIVE_YETUS_VERSION}-bin.tar"
+YETUS_BASEURL="https://archive.apache.org/dist/yetus/${HIVE_YETUS_VERSION}/;
+YETUS_TARBALL="yetus-${HIVE_YETUS_VERSION}-bin.tar"
 
 GPGBIN=$(command -v gpg)
-CURLBIN=$(command -v curl)
 
 pushd "${HIVE_PATCHPROCESS}" >/dev/null
 if [[ $? != 0 ]]; then
@@ -115,9 +142,9 @@ if [[ $? != 0 ]]; then
 fi
 
 if [[ -n "${CURLBIN}" ]]; then
-  "${CURLBIN}" -f -s -L -O "${BASEURL}/${TARBALL}.gz"
+  "${CURLBIN}" -f -s -L -O "${YETUS_BASEURL}/${YETUS_TARBALL}.gz"
   if [[ $? != 0 ]]; then
-yetus_error "ERROR: yetus-dl: unable to download ${BASEURL}/${TARBALL}.gz"
+yetus_error "ERROR: yetus-dl: unable to download 
${YETUS_BASEURL}/${YETUS_TARBALL}.gz"
 exit 1
   fi
 else
@@ -141,9 +168,9 @@ if [[ -n "${GPGBIN}" ]]; then
 yetus_error "ERROR: yetus-dl: unable to fetch 
https://dist.apache.org/repos/dist/release/yetus/KEYS;
 exit 1
   fi
-  "${CURLBIN}" -s -L -O "${BASEURL}/${TARBALL}.gz.asc"
+  "${CURLBIN}" -s -L -O "${YETUS_BASEURL}/${YETUS_TARBALL}.gz.asc"
   if [[ $? != 0 ]]; then
-yetus_error "ERROR: yetus-dl: unable to fetch ${BASEURL}/${TARBALL}.gz.asc"
+yetus_error "ERROR: yetus-dl: unable to fetch 
${YETUS_BASEURL}/${YETUS_TARBALL}.gz.asc"
 exit 1
   fi
   "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --import 
"${HIVE_PATCHPROCESS}/KEYS_YETUS" >/dev/null 2>&1
@@ -151,16 +178,16 @@ if [[ -n "${GPGBIN}" ]]; then
 yetus_error "ERROR: yetus-dl: gpg unable to import 
${HIVE_PATCHPROCESS}/KEYS_YETUS"
 exit 1
   fi
-  "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --verify 
"${TARBALL}.gz.asc" >/dev/null 2>&1
+  "${GPGBIN}" --homedir "${HIVE_PATCHPROCESS}/.gpg" --verify 
"${YETUS_TARBALL}.gz.asc" >/dev/null 2>&1
if [[ $? != 0 ]]; then
  yetus_error "ERROR: yetus-dl: gpg verify of tarball in 
${HIVE_PATCHPROCESS} failed"
  exit 1
fi
 fi
 
-gunzip -c "${TARBALL}.gz" | tar xpf -
+gunzip -c "${YETUS_TARBALL}.gz" | tar xpf -
 if [[ $? != 0 ]]; then
-  yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove 
${HIVE_PATCHPROCESS} to try again."
+  yetus_error "ERROR: ${YETUS_TARBALL}.gz is corrupt. Investigate and then 
remove ${HIVE_PATCHPROCESS} to try again."
   exit 1
 fi
 

http://git-wip-us.apache.org/repos/asf/hive/blob/42187fdb/standalone-metastore/findbugs/findbugs-exclude.xml

hive git commit: HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg, reviewed by Zoltan Haindrich)

[vgarg]

Repository: hive
Updated Branches:
  refs/heads/branch-3 0821efcad -> f57c33607


HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg,reviewed by 
Zoltan Haindrich)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f57c3360
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f57c3360
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f57c3360

Branch: refs/heads/branch-3
Commit: f57c336072ec5ae9af1b5b624eb4ec516ccdad0e
Parents: 0821efc
Author: Vineet Garg 
Authored: Wed Apr 11 11:17:36 2018 -0700
Committer: Vineet Garg 
Committed: Wed Apr 11 11:21:51 2018 -0700

--
 ql/src/test/queries/clientpositive/sysdb.q  |   5 +-
 .../results/clientpositive/llap/sysdb.q.out | 140 ++-
 2 files changed, 12 insertions(+), 133 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f57c3360/ql/src/test/queries/clientpositive/sysdb.q
--
diff --git a/ql/src/test/queries/clientpositive/sysdb.q 
b/ql/src/test/queries/clientpositive/sysdb.q
index 36d80e2..399c3ce 100644
--- a/ql/src/test/queries/clientpositive/sysdb.q
+++ b/ql/src/test/queries/clientpositive/sysdb.q
@@ -3,6 +3,7 @@ set hive.strict.checks.cartesian.product=false;
 set hive.compute.query.using.stats=false;
 
 set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 
 set hive.cbo.enable=false;
 
@@ -42,10 +43,6 @@ select grantor, principal_name from db_privs order by 
grantor, principal_name li
 
 select grantor, principal_name from global_privs order by grantor, 
principal_name limit 5;
 
-select index_name, index_handler_class from idxs order by index_name limit 5;
-
-select param_key, param_value from index_params order by param_key, 
param_value limit 5;
-
 select part_name from partitions order by part_name limit 5;
 
 select pkey_name, pkey_type from partition_keys order by pkey_name limit 5;

http://git-wip-us.apache.org/repos/asf/hive/blob/f57c3360/ql/src/test/results/clientpositive/llap/sysdb.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out 
b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index 73c5077..7d977d3 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -482,110 +482,6 @@ FROM
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: SYS@GLOBAL_PRIVS
 POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` (
-  `INDEX_ID` bigint,
-  `CREATE_TIME` int,
-  `DEFERRED_REBUILD` boolean,
-  `INDEX_HANDLER_CLASS` string,
-  `INDEX_NAME` string,
-  `INDEX_TBL_ID` bigint,
-  `LAST_ACCESS_TIME` int,
-  `ORIG_TBL_ID` bigint,
-  `SD_ID` bigint,
-  CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"CREATE_TIME\",
-  \"DEFERRED_REBUILD\",
-  \"INDEX_HANDLER_CLASS\",
-  \"INDEX_NAME\",
-  \"INDEX_TBL_ID\",
-  \"LAST_ACCESS_TIME\",
-  \"ORIG_TBL_ID\",
-  \"SD_ID\"
-FROM
-  \"IDXS\""
-)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: SYS@IDXS
-PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` (
-  `INDEX_ID` bigint,
-  `CREATE_TIME` int,
-  `DEFERRED_REBUILD` boolean,
-  `INDEX_HANDLER_CLASS` string,
-  `INDEX_NAME` string,
-  `INDEX_TBL_ID` bigint,
-  `LAST_ACCESS_TIME` int,
-  `ORIG_TBL_ID` bigint,
-  `SD_ID` bigint,
-  CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"CREATE_TIME\",
-  \"DEFERRED_REBUILD\",
-  \"INDEX_HANDLER_CLASS\",
-  \"INDEX_NAME\",
-  \"INDEX_TBL_ID\",
-  \"LAST_ACCESS_TIME\",
-  \"ORIG_TBL_ID\",
-  \"SD_ID\"
-FROM
-  \"IDXS\""
-)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: SYS@IDXS
-POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` (
-  `INDEX_ID` bigint,
-  `PARAM_KEY` string,
-  `PARAM_VALUE` string,
-  CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"PARAM_KEY\",
-  \"PARAM_VALUE\"
-FROM
-  \"INDEX_PARAMS\""
-)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: SYS@INDEX_PARAMS
-PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` (
-  `INDEX_ID` bigint,
-  `PARAM_KEY` string,
-  

hive git commit: HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg, reviewed by Zoltan Haindrich)

[vgarg]

Repository: hive
Updated Branches:
  refs/heads/master f8a671d8c -> 8777125b3


HIVE-19166: TestMiniLlapLocalCliDriver sysdb failure(Vineet Garg,reviewed by 
Zoltan Haindrich)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8777125b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8777125b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8777125b

Branch: refs/heads/master
Commit: 8777125b3d9dfcea7b7ce31f6535c5e64780bfbd
Parents: f8a671d
Author: Vineet Garg 
Authored: Wed Apr 11 11:17:36 2018 -0700
Committer: Vineet Garg 
Committed: Wed Apr 11 11:18:06 2018 -0700

--
 ql/src/test/queries/clientpositive/sysdb.q  |   5 +-
 .../results/clientpositive/llap/sysdb.q.out | 140 ++-
 2 files changed, 12 insertions(+), 133 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8777125b/ql/src/test/queries/clientpositive/sysdb.q
--
diff --git a/ql/src/test/queries/clientpositive/sysdb.q 
b/ql/src/test/queries/clientpositive/sysdb.q
index 36d80e2..399c3ce 100644
--- a/ql/src/test/queries/clientpositive/sysdb.q
+++ b/ql/src/test/queries/clientpositive/sysdb.q
@@ -3,6 +3,7 @@ set hive.strict.checks.cartesian.product=false;
 set hive.compute.query.using.stats=false;
 
 set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 
 set hive.cbo.enable=false;
 
@@ -42,10 +43,6 @@ select grantor, principal_name from db_privs order by 
grantor, principal_name li
 
 select grantor, principal_name from global_privs order by grantor, 
principal_name limit 5;
 
-select index_name, index_handler_class from idxs order by index_name limit 5;
-
-select param_key, param_value from index_params order by param_key, 
param_value limit 5;
-
 select part_name from partitions order by part_name limit 5;
 
 select pkey_name, pkey_type from partition_keys order by pkey_name limit 5;

http://git-wip-us.apache.org/repos/asf/hive/blob/8777125b/ql/src/test/results/clientpositive/llap/sysdb.q.out
--
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out 
b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index 73c5077..7d977d3 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -482,110 +482,6 @@ FROM
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: SYS@GLOBAL_PRIVS
 POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` (
-  `INDEX_ID` bigint,
-  `CREATE_TIME` int,
-  `DEFERRED_REBUILD` boolean,
-  `INDEX_HANDLER_CLASS` string,
-  `INDEX_NAME` string,
-  `INDEX_TBL_ID` bigint,
-  `LAST_ACCESS_TIME` int,
-  `ORIG_TBL_ID` bigint,
-  `SD_ID` bigint,
-  CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"CREATE_TIME\",
-  \"DEFERRED_REBUILD\",
-  \"INDEX_HANDLER_CLASS\",
-  \"INDEX_NAME\",
-  \"INDEX_TBL_ID\",
-  \"LAST_ACCESS_TIME\",
-  \"ORIG_TBL_ID\",
-  \"SD_ID\"
-FROM
-  \"IDXS\""
-)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: SYS@IDXS
-PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE TABLE IF NOT EXISTS `IDXS` (
-  `INDEX_ID` bigint,
-  `CREATE_TIME` int,
-  `DEFERRED_REBUILD` boolean,
-  `INDEX_HANDLER_CLASS` string,
-  `INDEX_NAME` string,
-  `INDEX_TBL_ID` bigint,
-  `LAST_ACCESS_TIME` int,
-  `ORIG_TBL_ID` bigint,
-  `SD_ID` bigint,
-  CONSTRAINT `SYS_PK_IDXS` PRIMARY KEY (`INDEX_ID`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"CREATE_TIME\",
-  \"DEFERRED_REBUILD\",
-  \"INDEX_HANDLER_CLASS\",
-  \"INDEX_NAME\",
-  \"INDEX_TBL_ID\",
-  \"LAST_ACCESS_TIME\",
-  \"ORIG_TBL_ID\",
-  \"SD_ID\"
-FROM
-  \"IDXS\""
-)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: SYS@IDXS
-POSTHOOK: Output: database:sys
-PREHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` (
-  `INDEX_ID` bigint,
-  `PARAM_KEY` string,
-  `PARAM_VALUE` string,
-  CONSTRAINT `SYS_PK_INDEX_PARAMS` PRIMARY KEY (`INDEX_ID`,`PARAM_KEY`) DISABLE
-)
-STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
-TBLPROPERTIES (
-"hive.sql.database.type" = "METASTORE",
-"hive.sql.query" =
-"SELECT
-  \"INDEX_ID\",
-  \"PARAM_KEY\",
-  \"PARAM_VALUE\"
-FROM
-  \"INDEX_PARAMS\""
-)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: SYS@INDEX_PARAMS
-PREHOOK: Output: database:sys
-POSTHOOK: query: CREATE TABLE IF NOT EXISTS `INDEX_PARAMS` (
-  `INDEX_ID` bigint,
-  `PARAM_KEY` string,
-  

hive git commit: HIVE-19077: Handle duplicate ptests requests standing in queue at the same time - addendum

[pvary]

Repository: hive
Updated Branches:
  refs/heads/master 7c1f01004 -> f8a671d8c


HIVE-19077: Handle duplicate ptests requests standing in queue at the same time 
- addendum


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8a671d8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8a671d8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8a671d8

Branch: refs/heads/master
Commit: f8a671d8cfe8a26d1d12c51f93207ec92577c796
Parents: 7c1f010
Author: Adam Szita 
Authored: Wed Apr 11 10:49:24 2018 +0200
Committer: Peter Vary 
Committed: Wed Apr 11 10:49:24 2018 +0200

--
 .../hive/ptest/api/client/JenkinsQueueUtil.java   | 18 ++
 1 file changed, 14 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/f8a671d8/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
--
diff --git 
a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
 
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
index f973781..f335164 100644
--- 
a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
+++ 
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
@@ -20,6 +20,8 @@
 package org.apache.hive.ptest.api.client;
 
 import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -27,7 +29,9 @@ import org.apache.commons.cli.CommandLine;
 import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -50,7 +54,6 @@ public class JenkinsQueueUtil {
   private static final String ISSUE_FIELD_KEY = "ISSUE_NUM";
   private static final String JIRA_KEY_PREFIX = "HIVE-";
 
-
   /**
* Looks up the current queue of the precommit job on a jenkins instance 
(specified by
* PTestClient.JENKINS_QUEUE_URL), and checks if current Jira is standing in 
queue already (i.e.
@@ -112,11 +115,15 @@ public class JenkinsQueueUtil {
 
   private static String httpGet(String url)
   throws IOException {
-DefaultHttpClient httpClient = new DefaultHttpClient();
+
 HttpGet request = new HttpGet(url);
 try {
+  CloseableHttpClient httpClient = HttpClientBuilder
+  .create()
+  
.setSslcontext(SSLContexts.custom().useProtocol("TLSv1.2").build())
+  .setRetryHandler(new PTestClient.PTestHttpRequestRetryHandler())
+  .build();
   request.addHeader("content-type", "application/json");
-  httpClient.setHttpRequestRetryHandler(new 
PTestClient.PTestHttpRequestRetryHandler());
   HttpResponse httpResponse = httpClient.execute(request);
   StatusLine statusLine = httpResponse.getStatusLine();
   if (statusLine.getStatusCode() != 200) {
@@ -124,6 +131,9 @@ public class JenkinsQueueUtil {
   }
   String response = EntityUtils.toString(httpResponse.getEntity(), 
"UTF-8");
   return response;
+} catch (NoSuchAlgorithmException | KeyManagementException e) {
+  e.printStackTrace();
+  throw new IOException(e.getMessage());
 } finally {
   request.abort();
 }

hive git commit: HIVE-19077: Handle duplicate ptests requests standing in queue at the same time (Adam Szita, via Peter Vary)

[pvary]

Repository: hive
Updated Branches:
  refs/heads/master f6dbcac35 -> 7c1f01004


HIVE-19077: Handle duplicate ptests requests standing in queue at the same time 
(Adam Szita, via Peter Vary)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7c1f0100
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7c1f0100
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7c1f0100

Branch: refs/heads/master
Commit: 7c1f010045db3af6fa5b805ae4f49fc9b07d5639
Parents: f6dbcac
Author: Adam Szita 
Authored: Wed Apr 11 09:45:56 2018 +0200
Committer: Peter Vary 
Committed: Wed Apr 11 09:45:56 2018 +0200

--
 dev-support/jenkins-common.sh   |   2 +
 dev-support/jenkins-execute-build.sh|   3 +-
 .../hive/ptest/api/client/JenkinsQueueUtil.java | 133 +++
 .../hive/ptest/api/client/PTestClient.java  |  15 ++-
 4 files changed, 150 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-common.sh
--
diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh
index 0467d11..64f486f 100644
--- a/dev-support/jenkins-common.sh
+++ b/dev-support/jenkins-common.sh
@@ -15,6 +15,8 @@
 # limitations under the License.
 
 JIRA_ROOT_URL="https://issues.apache.org;
+JENKINS_URL="https://builds.apache.org;
+JENKINS_QUEUE_QUERY="/queue/api/json?tree=items[task[name],inQueueSince,actions[parameters[name,value]],why]"
 
 fail() {
   echo "$@" 1>&2

http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/dev-support/jenkins-execute-build.sh
--
diff --git a/dev-support/jenkins-execute-build.sh 
b/dev-support/jenkins-execute-build.sh
index f660fcb..35392dd 100644
--- a/dev-support/jenkins-execute-build.sh
+++ b/dev-support/jenkins-execute-build.sh
@@ -51,7 +51,8 @@ call_ptest_server() {
local 
PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-3.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*"
 
java -cp "$PTEST_CLASSPATH" 
org.apache.hive.ptest.api.client.PTestClient --command testStart \
-   --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" 
--password "$JIRA_PASSWORD" "$@"
+   --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" 
--password "$JIRA_PASSWORD" \
+   --jenkinsQueueUrl "$JENKINS_URL$JENKINS_QUEUE_QUERY" "$@"
 }
 
 # Unpack all test results

http://git-wip-us.apache.org/repos/asf/hive/blob/7c1f0100/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
--
diff --git 
a/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
 
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
new file mode 100644
index 000..f973781
--- /dev/null
+++ 
b/testutils/ptest2/src/main/java/org/apache/hive/ptest/api/client/JenkinsQueueUtil.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hive.ptest.api.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.util.EntityUtils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+
+/**
+ * Utility class for the Precommit test job queue on Jenkins
+ */
+public class JenkinsQueueUtil {
+
+  private static final String JSON_ITEMS_FIELD = "items";
+  private static final String JSON_TASK_FIELD = "task";
+  private static final String JSON_TASK_NAME_FIELD = "name";
+