hive git commit: HIVE-20094 : Update Druid to 0.12.1 version (Slim Bouguerra via Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 19c2ec485 -> 6ef4a990d
HIVE-20094 : Update Druid to 0.12.1 version (Slim Bouguerra via Ashutosh
Chauhan)
Signed-off-by: Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6ef4a990
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6ef4a990
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6ef4a990
Branch: refs/heads/master
Commit: 6ef4a990db9a9c3fd890b3e7f8e810e032c5ab2c
Parents: 19c2ec4
Author: Slim Bouguerra
Authored: Wed Jul 11 22:48:23 2018 -0700
Committer: Ashutosh Chauhan
Committed: Wed Jul 11 22:48:23 2018 -0700
--
.../java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java| 2 +-
pom.xml| 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/6ef4a990/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
--
diff --git
a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
index d1f0d98..400262a 100644
---
a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
+++
b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
@@ -181,7 +181,7 @@ public class DruidRecordWriter implements
RecordWriter segmentsToPush) {
try {
SegmentsAndMetadata segmentsAndMetadata = appenderator
- .push(segmentsToPush, committerSupplier.get()).get();
+ .push(segmentsToPush, committerSupplier.get(), false).get();
final HashSet pushedSegmentIdentifierHashSet = new HashSet<>();
for (DataSegment pushedSegment : segmentsAndMetadata.getSegments()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6ef4a990/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 28ad152..604ee76 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,7 +143,7 @@
10.14.1.0
3.1.0
0.1.2
-0.12.0
+0.12.1
1.2.0-3f79e055
19.0
2.4.11
[2/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out index 0967fde..61508b5 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out @@ -15,21 +15,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n10 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n10 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n10 PREHOOK: type: QUERY @@ -39,506 +55,12 @@ POSTHOOK: query: select * from masking_test_n10 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n10 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284
[9/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fde4440
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fde4440
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fde4440
Branch: refs/heads/branch-3.1
Commit: 3fde4440baec9f6ea55eb0c85bb4d3bcbd17f9a2
Parents: c8796ad
Author: Jesus Camacho Rodriguez
Authored: Mon Jul 9 11:42:26 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Jul 11 16:32:08 2018 -0700
--
...SQLStdHiveAuthorizationValidatorForTest.java | 12 +-
.../llap/results_cache_with_masking.q.out | 625 +-
.../test/results/clientpositive/masking_1.q.out | 7252 +
.../results/clientpositive/masking_12.q.out | 1742 +
.../clientpositive/masking_1_newdb.q.out| 1050 +--
.../test/results/clientpositive/masking_2.q.out | 1128 +--
.../test/results/clientpositive/masking_3.q.out | 6420 +---
.../test/results/clientpositive/masking_4.q.out | 156 +-
.../test/results/clientpositive/masking_5.q.out | 833 +-
.../test/results/clientpositive/masking_6.q.out | 2062 +
.../test/results/clientpositive/masking_7.q.out | 2062 +
.../test/results/clientpositive/masking_8.q.out | 1641 +---
.../clientpositive/masking_disablecbo_1.q.out | 7264 +-
.../clientpositive/masking_disablecbo_2.q.out | 1223 +--
.../clientpositive/masking_disablecbo_4.q.out | 84 +-
.../clientpositive/union_pos_alias.q.out| 1652 +---
16 files changed, 4860 insertions(+), 30346 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
index d883e4b..92c1292 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
@@ -119,7 +119,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
List privObjs) throws SemanticException {
List needRewritePrivObjs = new ArrayList<>();
for (HivePrivilegeObject privObj : privObjs) {
- if (privObj.getObjectName().equals("masking_test")) {
+ if (privObj.getObjectName().equals("masking_test") ||
privObj.getObjectName().startsWith("masking_test_n")) {
privObj.setRowFilterExpression("key % 2 = 0 and key < 10");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -131,7 +131,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_view")) {
+ } else if (privObj.getObjectName().equals("masking_test_view") ||
privObj.getObjectName().startsWith("masking_test_view_n")) {
privObj.setRowFilterExpression("key > 6");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -143,14 +143,14 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_subq")) {
+ } else if (privObj.getObjectName().equals("masking_test_subq") ||
privObj.getObjectName().startsWith("masking_test_subq_n")) {
privObj
-.setRowFilterExpression("key in (select key from src where src.key
= masking_test_subq.key)");
+.setRowFilterExpression("key in (select key from src where src.key
= " + privObj.getObjectName() + ".key)");
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_acid_no_masking")) {
+ } else if (privObj.getObjectName().equals("masking_acid_no_masking") ||
privObj.getObjectName().startsWith("masking_acid_no_masking_n")) {
// testing acid usage when no masking/filtering is present
needRewritePrivObjs.add(privObj);
- } else if
[3/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_8.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_8.q.out b/ql/src/test/results/clientpositive/masking_8.q.out index cc4d155..d602ec5 100644 --- a/ql/src/test/results/clientpositive/masking_8.q.out +++ b/ql/src/test/results/clientpositive/masking_8.q.out @@ -20,21 +20,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n2 +Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string), _c2 (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n2 - Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string), _c2 (type: string), ROW__ID (type: struct) -outputColumnNames: _col0, _col1, _col2, _col3 -Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select *, ROW__ID from masking_test_n2 PREHOOK: type: QUERY @@ -44,526 +60,48 @@ POSTHOOK: query: select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n2 A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12 NULL -195val_195 12 NULL -475val_475 12 NULL -17
[6/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_3.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_3.q.out b/ql/src/test/results/clientpositive/masking_3.q.out index 8553b08..1bd14bc 100644 --- a/ql/src/test/results/clientpositive/masking_3.q.out +++ b/ql/src/test/results/clientpositive/masking_3.q.out @@ -15,539 +15,741 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_subq_n3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: +Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + TableScan +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Reduce Operator Tree: +Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 UDFToDouble(_col0) (type: double) +1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: _col0 (type: string), _col1 (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type:
[4/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_7.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_7.q.out b/ql/src/test/results/clientpositive/masking_7.q.out index 9fa62f9..6251f0c 100644 --- a/ql/src/test/results/clientpositive/masking_7.q.out +++ b/ql/src/test/results/clientpositive/masking_7.q.out @@ -33,17 +33,20 @@ STAGE PLANS: properties: insideView TRUE Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), ROW__ID (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -62,506 +65,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n14 POSTHOOK: Input: default@src A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12 NULL -195val_195 12 NULL -475val_475 12 NULL -17 val_17 12 NULL -113val_113 12 NULL -155val_155 12 NULL -203val_203 12 NULL -339val_339
[1/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/branch-3.1 c8796addd -> 3fde4440b http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out index bb58b08..4ce94b7 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out @@ -15,21 +15,41 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n13 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: UDFToInteger(key) (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n13 PREHOOK: type: QUERY @@ -41,506 +61,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n13 POSTHOOK: Input: default@src A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51
[5/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_4.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_4.q.out b/ql/src/test/results/clientpositive/masking_4.q.out index 068c484..1d096f8 100644 --- a/ql/src/test/results/clientpositive/masking_4.q.out +++ b/ql/src/test/results/clientpositive/masking_4.q.out @@ -85,15 +85,15 @@ STAGE PLANS: alias: masking_test_n11 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -126,15 +126,15 @@ STAGE PLANS: alias: masking_test_n11 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -156,19 +156,137 @@ with q1 as ( select * from masking_test_n11 where key = '5') select * from masking_test_subq_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n2 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: +Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +
[7/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_12.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_12.q.out b/ql/src/test/results/clientpositive/masking_12.q.out index 32c420c..cc49fd3 100644 --- a/ql/src/test/results/clientpositive/masking_12.q.out +++ b/ql/src/test/results/clientpositive/masking_12.q.out @@ -29,23 +29,39 @@ POSTHOOK: query: explain select * from `v0` POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n5 +properties: + insideView TRUE +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n5 - properties: -insideView TRUE - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from `v0` PREHOOK: type: QUERY @@ -57,506 +73,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n5 POSTHOOK: Input: default@v0 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284 -12
[8/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3fde4440/ql/src/test/results/clientpositive/masking_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_1.q.out b/ql/src/test/results/clientpositive/masking_1.q.out index bc0a42a..6e0a9eb 100644 --- a/ql/src/test/results/clientpositive/masking_1.q.out +++ b/ql/src/test/results/clientpositive/masking_1.q.out @@ -15,21 +15,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n8 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n8 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n8 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n8 PREHOOK: type: QUERY @@ -39,506 +55,12 @@ POSTHOOK: query: select * from masking_test_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n8 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284 -12 val_12 -230val_230 -181val_181 -67 val_67
[6/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_3.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_3.q.out b/ql/src/test/results/clientpositive/masking_3.q.out index 8553b08..1bd14bc 100644 --- a/ql/src/test/results/clientpositive/masking_3.q.out +++ b/ql/src/test/results/clientpositive/masking_3.q.out @@ -15,539 +15,741 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_subq_n3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: +Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + TableScan +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Reduce Operator Tree: +Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 UDFToDouble(_col0) (type: double) +1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: _col0 (type: string), _col1 (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type:
[2/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out index 0967fde..61508b5 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out @@ -15,21 +15,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n10 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n10 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n10 PREHOOK: type: QUERY @@ -39,506 +55,12 @@ POSTHOOK: query: select * from masking_test_n10 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n10 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284
[4/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_7.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_7.q.out b/ql/src/test/results/clientpositive/masking_7.q.out index 9fa62f9..6251f0c 100644 --- a/ql/src/test/results/clientpositive/masking_7.q.out +++ b/ql/src/test/results/clientpositive/masking_7.q.out @@ -33,17 +33,20 @@ STAGE PLANS: properties: insideView TRUE Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), ROW__ID (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -62,506 +65,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n14 POSTHOOK: Input: default@src A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12 NULL -195val_195 12 NULL -475val_475 12 NULL -17 val_17 12 NULL -113val_113 12 NULL -155val_155 12 NULL -203val_203 12 NULL -339val_339
[9/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c71687d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c71687d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c71687d
Branch: refs/heads/branch-3
Commit: 3c71687d1b7bc180ac303465fb45b7efef99a08b
Parents: b7834e6
Author: Jesus Camacho Rodriguez
Authored: Mon Jul 9 11:42:26 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Jul 11 16:31:45 2018 -0700
--
...SQLStdHiveAuthorizationValidatorForTest.java | 12 +-
.../llap/results_cache_with_masking.q.out | 625 +-
.../test/results/clientpositive/masking_1.q.out | 7252 +
.../results/clientpositive/masking_12.q.out | 1742 +
.../clientpositive/masking_1_newdb.q.out| 1050 +--
.../test/results/clientpositive/masking_2.q.out | 1128 +--
.../test/results/clientpositive/masking_3.q.out | 6420 +---
.../test/results/clientpositive/masking_4.q.out | 156 +-
.../test/results/clientpositive/masking_5.q.out | 833 +-
.../test/results/clientpositive/masking_6.q.out | 2062 +
.../test/results/clientpositive/masking_7.q.out | 2062 +
.../test/results/clientpositive/masking_8.q.out | 1641 +---
.../clientpositive/masking_disablecbo_1.q.out | 7264 +-
.../clientpositive/masking_disablecbo_2.q.out | 1223 +--
.../clientpositive/masking_disablecbo_4.q.out | 84 +-
.../clientpositive/union_pos_alias.q.out| 1652 +---
16 files changed, 4860 insertions(+), 30346 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
index d883e4b..92c1292 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
@@ -119,7 +119,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
List privObjs) throws SemanticException {
List needRewritePrivObjs = new ArrayList<>();
for (HivePrivilegeObject privObj : privObjs) {
- if (privObj.getObjectName().equals("masking_test")) {
+ if (privObj.getObjectName().equals("masking_test") ||
privObj.getObjectName().startsWith("masking_test_n")) {
privObj.setRowFilterExpression("key % 2 = 0 and key < 10");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -131,7 +131,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_view")) {
+ } else if (privObj.getObjectName().equals("masking_test_view") ||
privObj.getObjectName().startsWith("masking_test_view_n")) {
privObj.setRowFilterExpression("key > 6");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -143,14 +143,14 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_subq")) {
+ } else if (privObj.getObjectName().equals("masking_test_subq") ||
privObj.getObjectName().startsWith("masking_test_subq_n")) {
privObj
-.setRowFilterExpression("key in (select key from src where src.key
= masking_test_subq.key)");
+.setRowFilterExpression("key in (select key from src where src.key
= " + privObj.getObjectName() + ".key)");
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_acid_no_masking")) {
+ } else if (privObj.getObjectName().equals("masking_acid_no_masking") ||
privObj.getObjectName().startsWith("masking_acid_no_masking_n")) {
// testing acid usage when no masking/filtering is present
needRewritePrivObjs.add(privObj);
- } else if
[5/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_4.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_4.q.out b/ql/src/test/results/clientpositive/masking_4.q.out index 068c484..1d096f8 100644 --- a/ql/src/test/results/clientpositive/masking_4.q.out +++ b/ql/src/test/results/clientpositive/masking_4.q.out @@ -85,15 +85,15 @@ STAGE PLANS: alias: masking_test_n11 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -126,15 +126,15 @@ STAGE PLANS: alias: masking_test_n11 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -156,19 +156,137 @@ with q1 as ( select * from masking_test_n11 where key = '5') select * from masking_test_subq_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n2 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: +Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +
[8/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_1.q.out b/ql/src/test/results/clientpositive/masking_1.q.out index bc0a42a..6e0a9eb 100644 --- a/ql/src/test/results/clientpositive/masking_1.q.out +++ b/ql/src/test/results/clientpositive/masking_1.q.out @@ -15,21 +15,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n8 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n8 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n8 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n8 PREHOOK: type: QUERY @@ -39,506 +55,12 @@ POSTHOOK: query: select * from masking_test_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n8 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284 -12 val_12 -230val_230 -181val_181 -67 val_67
[3/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_8.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_8.q.out b/ql/src/test/results/clientpositive/masking_8.q.out index cc4d155..d602ec5 100644 --- a/ql/src/test/results/clientpositive/masking_8.q.out +++ b/ql/src/test/results/clientpositive/masking_8.q.out @@ -20,21 +20,37 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n2 +Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string), _c2 (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n2 - Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string), _c2 (type: string), ROW__ID (type: struct) -outputColumnNames: _col0, _col1, _col2, _col3 -Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select *, ROW__ID from masking_test_n2 PREHOOK: type: QUERY @@ -44,526 +60,48 @@ POSTHOOK: query: select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n2 A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12 NULL -195val_195 12 NULL -475val_475 12 NULL -17
[1/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/branch-3 b7834e679 -> 3c71687d1 http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out index bb58b08..4ce94b7 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out @@ -15,21 +15,41 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n13 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: UDFToInteger(key) (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n13 PREHOOK: type: QUERY @@ -41,506 +61,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n13 POSTHOOK: Input: default@src A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51
[7/9] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/3c71687d/ql/src/test/results/clientpositive/masking_12.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_12.q.out b/ql/src/test/results/clientpositive/masking_12.q.out index 32c420c..cc49fd3 100644 --- a/ql/src/test/results/clientpositive/masking_12.q.out +++ b/ql/src/test/results/clientpositive/masking_12.q.out @@ -29,23 +29,39 @@ POSTHOOK: query: explain select * from `v0` POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n5 +properties: + insideView TRUE +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n5 - properties: -insideView TRUE - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from `v0` PREHOOK: type: QUERY @@ -57,506 +73,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n5 POSTHOOK: Input: default@v0 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284val_284 -12
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum)
Repository: hive Updated Branches: refs/heads/branch-3.1 7e649028c -> c8796addd HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c8796add Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c8796add Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c8796add Branch: refs/heads/branch-3.1 Commit: c8796a27ca94382871d2f838050cf5e4d454 Parents: 7e64902 Author: Teddy Choi Authored: Thu Jul 12 08:13:27 2018 +0900 Committer: Teddy Choi Committed: Thu Jul 12 08:15:05 2018 +0900 -- .../queries/clientpositive/vector_delete_orig_table.q| 2 ++ .../clientpositive/vector_delete_orig_table.q.out| 11 ++- 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/c8796add/ql/src/test/queries/clientpositive/vector_delete_orig_table.q -- diff --git a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q index f914408..48ef5e2 100644 --- a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q +++ b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q @@ -29,6 +29,8 @@ select count(*) from acid_dot; select count(*) from acid_dot; +select count(*) from acid_dot where cint < -1070551679; + delete from acid_dot where cint < -1070551679; select count(*) from acid_dot; http://git-wip-us.apache.org/repos/asf/hive/blob/c8796add/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out index 5d7f310..60d3c4d 100644 --- a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out +++ b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out @@ -132,6 +132,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here 12288 +PREHOOK: query: select count(*) from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot + A masked pattern was here +POSTHOOK: query: select count(*) from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot + A masked pattern was here +8 PREHOOK: query: delete from acid_dot where cint < -1070551679 PREHOOK: type: QUERY PREHOOK: Input: default@acid_dot @@ -148,5 +157,5 @@ POSTHOOK: query: select count(*) from acid_dot POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here -12192 +12280 A masked pattern was here
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum)
Repository: hive Updated Branches: refs/heads/branch-3 c829f61cb -> b7834e679 HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b7834e67 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b7834e67 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b7834e67 Branch: refs/heads/branch-3 Commit: b7834e679dbef5774b74d05b0136b888d1ff3ff3 Parents: c829f61 Author: Teddy Choi Authored: Thu Jul 12 08:13:27 2018 +0900 Committer: Teddy Choi Committed: Thu Jul 12 08:14:27 2018 +0900 -- .../queries/clientpositive/vector_delete_orig_table.q| 2 ++ .../clientpositive/vector_delete_orig_table.q.out| 11 ++- 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/b7834e67/ql/src/test/queries/clientpositive/vector_delete_orig_table.q -- diff --git a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q index f914408..48ef5e2 100644 --- a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q +++ b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q @@ -29,6 +29,8 @@ select count(*) from acid_dot; select count(*) from acid_dot; +select count(*) from acid_dot where cint < -1070551679; + delete from acid_dot where cint < -1070551679; select count(*) from acid_dot; http://git-wip-us.apache.org/repos/asf/hive/blob/b7834e67/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out index 5d7f310..60d3c4d 100644 --- a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out +++ b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out @@ -132,6 +132,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here 12288 +PREHOOK: query: select count(*) from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot + A masked pattern was here +POSTHOOK: query: select count(*) from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot + A masked pattern was here +8 PREHOOK: query: delete from acid_dot where cint < -1070551679 PREHOOK: type: QUERY PREHOOK: Input: default@acid_dot @@ -148,5 +157,5 @@ POSTHOOK: query: select count(*) from acid_dot POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here -12192 +12280 A masked pattern was here
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum)
Repository: hive Updated Branches: refs/heads/master 6340a81f4 -> 19c2ec485 HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman) (adduendum) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19c2ec48 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19c2ec48 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19c2ec48 Branch: refs/heads/master Commit: 19c2ec4855d4dd5baf512ef23262555431b6c538 Parents: 6340a81 Author: Teddy Choi Authored: Thu Jul 12 08:13:27 2018 +0900 Committer: Teddy Choi Committed: Thu Jul 12 08:13:27 2018 +0900 -- .../queries/clientpositive/vector_delete_orig_table.q| 2 ++ .../clientpositive/vector_delete_orig_table.q.out| 11 ++- 2 files changed, 12 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hive/blob/19c2ec48/ql/src/test/queries/clientpositive/vector_delete_orig_table.q -- diff --git a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q index f914408..48ef5e2 100644 --- a/ql/src/test/queries/clientpositive/vector_delete_orig_table.q +++ b/ql/src/test/queries/clientpositive/vector_delete_orig_table.q @@ -29,6 +29,8 @@ select count(*) from acid_dot; select count(*) from acid_dot; +select count(*) from acid_dot where cint < -1070551679; + delete from acid_dot where cint < -1070551679; select count(*) from acid_dot; http://git-wip-us.apache.org/repos/asf/hive/blob/19c2ec48/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out -- diff --git a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out index 5d7f310..60d3c4d 100644 --- a/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out +++ b/ql/src/test/results/clientpositive/vector_delete_orig_table.q.out @@ -132,6 +132,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here 12288 +PREHOOK: query: select count(*) from acid_dot where cint < -1070551679 +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_dot + A masked pattern was here +POSTHOOK: query: select count(*) from acid_dot where cint < -1070551679 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_dot + A masked pattern was here +8 PREHOOK: query: delete from acid_dot where cint < -1070551679 PREHOOK: type: QUERY PREHOOK: Input: default@acid_dot @@ -148,5 +157,5 @@ POSTHOOK: query: select count(*) from acid_dot POSTHOOK: type: QUERY POSTHOOK: Input: default@acid_dot A masked pattern was here -12192 +12280 A masked pattern was here
[07/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_3.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_3.q.out b/ql/src/test/results/clientpositive/masking_3.q.out index 20d9231..0dbc394 100644 --- a/ql/src/test/results/clientpositive/masking_3.q.out +++ b/ql/src/test/results/clientpositive/masking_3.q.out @@ -15,540 +15,665 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_subq_n3 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +filterExpr: key is not null (type: boolean) +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized + Reduce Operator Tree: +Group By Operator + keys: KEY._col0 (type: int) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 +Map Reduce + Map Operator Tree: + TableScan +alias: src +filterExpr: key is not null (type: boolean) +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: string) +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + TableScan +Reduce Output Operator + key expressions: UDFToDouble(_col0) (type: double) + sort order: + + Map-reduce partition columns: UDFToDouble(_col0) (type: double) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Reduce Operator Tree: +Join Operator + condition map: + Inner Join 0 to 1 + keys: +0 UDFToDouble(_col0) (type: double) +1 UDFToDouble(_col0) (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: _col0 (type: string), _col1 (type: int) +mode: hash +outputColumnNames: _col0, _col1 +Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n3 +filterExpr: key is not null (type: boolean) +
[10/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6340a81f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6340a81f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6340a81f
Branch: refs/heads/master
Commit: 6340a81f487c407f63cf2a00e39b2f905328d74e
Parents: 4a0814b
Author: Jesus Camacho Rodriguez
Authored: Mon Jul 9 11:42:26 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Jul 11 16:07:40 2018 -0700
--
...SQLStdHiveAuthorizationValidatorForTest.java | 12 +-
.../llap/results_cache_with_masking.q.out | 626 +-
.../test/results/clientpositive/masking_1.q.out | 7265 +
.../results/clientpositive/masking_12.q.out | 1752 +
.../clientpositive/masking_1_newdb.q.out| 1053 +--
.../test/results/clientpositive/masking_2.q.out | 1140 +--
.../test/results/clientpositive/masking_3.q.out | 6399 ---
.../test/results/clientpositive/masking_4.q.out | 163 +-
.../test/results/clientpositive/masking_5.q.out | 836 +-
.../test/results/clientpositive/masking_6.q.out | 2068 +
.../test/results/clientpositive/masking_7.q.out | 2068 +
.../test/results/clientpositive/masking_8.q.out | 1645 +---
.../clientpositive/masking_disablecbo_1.q.out | 7277 +-
.../clientpositive/masking_disablecbo_2.q.out | 1225 +--
.../clientpositive/masking_disablecbo_4.q.out | 90 +-
.../clientpositive/union_pos_alias.q.out| 1658 +---
16 files changed, 4916 insertions(+), 30361 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
--
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
index d883e4b..92c1292 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAuthorizationValidatorForTest.java
@@ -119,7 +119,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
List privObjs) throws SemanticException {
List needRewritePrivObjs = new ArrayList<>();
for (HivePrivilegeObject privObj : privObjs) {
- if (privObj.getObjectName().equals("masking_test")) {
+ if (privObj.getObjectName().equals("masking_test") ||
privObj.getObjectName().startsWith("masking_test_n")) {
privObj.setRowFilterExpression("key % 2 = 0 and key < 10");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -131,7 +131,7 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_view")) {
+ } else if (privObj.getObjectName().equals("masking_test_view") ||
privObj.getObjectName().startsWith("masking_test_view_n")) {
privObj.setRowFilterExpression("key > 6");
List cellValueTransformers = new ArrayList<>();
for (String columnName : privObj.getColumns()) {
@@ -143,14 +143,14 @@ public class SQLStdHiveAuthorizationValidatorForTest
extends SQLStdHiveAuthoriza
}
privObj.setCellValueTransformers(cellValueTransformers);
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_test_subq")) {
+ } else if (privObj.getObjectName().equals("masking_test_subq") ||
privObj.getObjectName().startsWith("masking_test_subq_n")) {
privObj
-.setRowFilterExpression("key in (select key from src where src.key
= masking_test_subq.key)");
+.setRowFilterExpression("key in (select key from src where src.key
= " + privObj.getObjectName() + ".key)");
needRewritePrivObjs.add(privObj);
- } else if (privObj.getObjectName().equals("masking_acid_no_masking")) {
+ } else if (privObj.getObjectName().equals("masking_acid_no_masking") ||
privObj.getObjectName().startsWith("masking_acid_no_masking_n")) {
// testing acid usage when no masking/filtering is present
needRewritePrivObjs.add(privObj);
- } else if
[02/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out index fa92660..cf7fd00 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_1.q.out @@ -15,21 +15,38 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n10 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n10 +filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean) +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n10 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n10 PREHOOK: type: QUERY @@ -39,506 +56,12 @@ POSTHOOK: query: select * from masking_test_n10 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n10 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241
[03/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_8.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_8.q.out b/ql/src/test/results/clientpositive/masking_8.q.out index cc4d155..c4590d3 100644 --- a/ql/src/test/results/clientpositive/masking_8.q.out +++ b/ql/src/test/results/clientpositive/masking_8.q.out @@ -20,21 +20,38 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n2 +filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean) +Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string), _c2 (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 1130 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n2 - Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string), _c2 (type: string), ROW__ID (type: struct) -outputColumnNames: _col0, _col1, _col2, _col3 -Statistics: Num rows: 500 Data size: 6812 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select *, ROW__ID from masking_test_n2 PREHOOK: type: QUERY @@ -44,526 +61,49 @@ POSTHOOK: query: select *, ROW__ID from masking_test_n2 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n2 A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12
[09/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_1.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_1.q.out b/ql/src/test/results/clientpositive/masking_1.q.out index 19d1fcc..b5e67e2 100644 --- a/ql/src/test/results/clientpositive/masking_1.q.out +++ b/ql/src/test/results/clientpositive/masking_1.q.out @@ -15,21 +15,38 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n8 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n8 +filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean) +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n8 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n8 PREHOOK: type: QUERY @@ -39,506 +56,12 @@ POSTHOOK: query: select * from masking_test_n8 POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n8 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241 -83 val_83 -333val_333 -180val_180 -284
[04/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_7.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_7.q.out b/ql/src/test/results/clientpositive/masking_7.q.out index 9727f5e..bbdf3f9 100644 --- a/ql/src/test/results/clientpositive/masking_7.q.out +++ b/ql/src/test/results/clientpositive/masking_7.q.out @@ -30,20 +30,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src +filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) properties: insideView TRUE Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string), ROW__ID (type: struct) - outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string), ROW__ID (type: struct) +outputColumnNames: _col0, _col1, _col2, _col3 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -62,506 +66,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n14 POSTHOOK: Input: default@src A masked pattern was here -238val_238 12 NULL -86 val_86 12 NULL -311val_311 12 NULL -27 val_27 12 NULL -165val_165 12 NULL -409val_409 12 NULL -255val_255 12 NULL -278val_278 12 NULL -98 val_98 12 NULL -484val_484 12 NULL -265val_265 12 NULL -193val_193 12 NULL -401val_401 12 NULL -150val_150 12 NULL -273val_273 12 NULL -224val_224 12 NULL -369val_369 12 NULL -66 val_66 12 NULL -128val_128 12 NULL -213val_213 12 NULL -146val_146 12 NULL -406val_406 12 NULL -429val_429 12 NULL -374val_374 12 NULL -152val_152 12 NULL -469val_469 12 NULL -145val_145 12 NULL -495val_495 12 NULL -37 val_37 12 NULL -327val_327 12 NULL -281val_281 12 NULL -277val_277 12 NULL -209val_209 12 NULL -15 val_15 12 NULL -82 val_82 12 NULL -403val_403 12 NULL -166val_166 12 NULL -417val_417 12 NULL -430val_430 12 NULL -252val_252 12 NULL -292val_292 12 NULL -219val_219 12 NULL -287val_287 12 NULL -153val_153 12 NULL -193val_193 12 NULL -338val_338 12 NULL -446val_446 12 NULL -459val_459 12 NULL -394val_394 12 NULL -237val_237 12 NULL -482val_482 12 NULL -174val_174 12 NULL -413val_413 12 NULL -494val_494 12 NULL -207val_207 12 NULL -199val_199 12 NULL -466val_466 12 NULL -208val_208 12 NULL -174val_174 12 NULL -399val_399 12 NULL -396val_396 12 NULL -247val_247 12 NULL -417val_417 12 NULL -489val_489 12 NULL -162val_162 12 NULL -377val_377 12 NULL -397val_397 12 NULL -309val_309 12 NULL -365val_365 12 NULL -266val_266 12 NULL -439val_439 12 NULL -342val_342 12 NULL -367val_367 12 NULL -325val_325 12 NULL -167val_167 12 NULL -195val_195
[05/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_6.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_6.q.out b/ql/src/test/results/clientpositive/masking_6.q.out index a8fab44..a98155a 100644 --- a/ql/src/test/results/clientpositive/masking_6.q.out +++ b/ql/src/test/results/clientpositive/masking_6.q.out @@ -29,20 +29,24 @@ STAGE PLANS: Map Operator Tree: TableScan alias: src +filterExpr: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) properties: insideView TRUE Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -Select Operator - expressions: UDFToInteger(key) (type: int), value (type: string), '12' (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator -compressed: false -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -table: -input format: org.apache.hadoop.mapred.SequenceFileInputFormat -output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat -serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +Filter Operator + predicate: (((UDFToInteger(key) % 2) = 0) and (UDFToInteger(key) < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: UDFToInteger(key) (type: int), reverse(value) (type: string), '12' (type: string) +outputColumnNames: _col0, _col1, _col2 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Execution mode: vectorized Stage: Stage-0 @@ -61,506 +65,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n0 POSTHOOK: Input: default@src A masked pattern was here -238val_238 12 -86 val_86 12 -311val_311 12 -27 val_27 12 -165val_165 12 -409val_409 12 -255val_255 12 -278val_278 12 -98 val_98 12 -484val_484 12 -265val_265 12 -193val_193 12 -401val_401 12 -150val_150 12 -273val_273 12 -224val_224 12 -369val_369 12 -66 val_66 12 -128val_128 12 -213val_213 12 -146val_146 12 -406val_406 12 -429val_429 12 -374val_374 12 -152val_152 12 -469val_469 12 -145val_145 12 -495val_495 12 -37 val_37 12 -327val_327 12 -281val_281 12 -277val_277 12 -209val_209 12 -15 val_15 12 -82 val_82 12 -403val_403 12 -166val_166 12 -417val_417 12 -430val_430 12 -252val_252 12 -292val_292 12 -219val_219 12 -287val_287 12 -153val_153 12 -193val_193 12 -338val_338 12 -446val_446 12 -459val_459 12 -394val_394 12 -237val_237 12 -482val_482 12 -174val_174 12 -413val_413 12 -494val_494 12 -207val_207 12 -199val_199 12 -466val_466 12 -208val_208 12 -174val_174 12 -399val_399 12 -396val_396 12 -247val_247 12 -417val_417 12 -489val_489 12 -162val_162 12 -377val_377 12 -397val_397 12 -309val_309 12 -365val_365 12 -266val_266 12 -439val_439 12 -342val_342 12 -367val_367 12 -325val_325 12 -167val_167 12 -195val_195 12 -475val_475 12 -17 val_17 12 -113val_113 12 -155val_155 12 -203val_203 12 -339val_339 12 -0 val_0 12 -455val_455 12 -128val_128 12 -311val_311 12 -316val_316 12 -57 val_57 12 -302val_302 12 -205val_205 12 -149val_149 12 -438val_438 12 -345val_345 12 -129val_129 12 -170val_170 12 -20 val_20 12 -489val_489 12 -157val_157 12 -378val_378 12 -221val_221 12 -92 val_92 12 -111val_111 12 -47 val_47 12 -72 val_72 12 -4 val_4 12 -280val_280 12 -35 val_35 12 -427val_427 12 -277val_277 12 -208val_208 12 -356val_356 12 -399val_399 12 -169val_169 12 -382val_382 12 -498val_498 12 -125val_125 12 -386val_386 12 -437val_437 12 -469
[06/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_4.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_4.q.out b/ql/src/test/results/clientpositive/masking_4.q.out index 5f90f3e..8ece717 100644 --- a/ql/src/test/results/clientpositive/masking_4.q.out +++ b/ql/src/test/results/clientpositive/masking_4.q.out @@ -84,18 +84,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: masking_test_n11 -filterExpr: (key = 5) (type: boolean) +filterExpr: (((key % 2) = 0) and (key = 5)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -126,18 +126,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: masking_test_n11 -filterExpr: (key = 5) (type: boolean) +filterExpr: (((key % 2) = 0) and (key = 5)) (type: boolean) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator - predicate: (key = 5) (type: boolean) - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + predicate: (((key % 2) = 0) and (key = 5)) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE Select Operator -expressions: 5 (type: int), value (type: string) +expressions: 5 (type: int), reverse(value) (type: string) outputColumnNames: _col0, _col1 -Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE +Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -159,19 +159,140 @@ with q1 as ( select * from masking_test_n11 where key = '5') select * from masking_test_subq_n2 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-3 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_subq_n2 +filterExpr: key is not null (type: boolean) +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator +keys: key (type: int) +mode: hash +outputColumnNames: _col0 +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column
[08/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_12.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_12.q.out b/ql/src/test/results/clientpositive/masking_12.q.out index bd9e596..15ff0ef 100644 --- a/ql/src/test/results/clientpositive/masking_12.q.out +++ b/ql/src/test/results/clientpositive/masking_12.q.out @@ -29,23 +29,40 @@ POSTHOOK: query: explain select * from `v0` POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: masking_test_n5 +filterExpr: (((key % 2) = 0) and (key < 10)) (type: boolean) +properties: + insideView TRUE +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Filter Operator + predicate: (((key % 2) = 0) and (key < 10)) (type: boolean) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator +expressions: key (type: int), reverse(value) (type: string) +outputColumnNames: _col0, _col1 +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: masking_test_n5 - properties: -insideView TRUE - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: key (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from `v0` PREHOOK: type: QUERY @@ -57,506 +74,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n5 POSTHOOK: Input: default@v0 A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51 val_51 -138val_138 -103val_103 -239val_239 -213val_213 -216val_216 -430val_430 -278val_278 -176val_176 -289val_289 -221val_221 -65 val_65 -318val_318 -332val_332 -311val_311 -275val_275 -137val_137 -241val_241
[01/10] hive git commit: HIVE-20123: Fix masking tests after HIVE-19617 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive Updated Branches: refs/heads/master 4a0814b7f -> 6340a81f4 http://git-wip-us.apache.org/repos/asf/hive/blob/6340a81f/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out -- diff --git a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out index dc53112..fad0120 100644 --- a/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out +++ b/ql/src/test/results/clientpositive/masking_disablecbo_2.q.out @@ -15,21 +15,41 @@ PREHOOK: type: QUERY POSTHOOK: query: explain select * from masking_test_n13 POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-0 is a root stage + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 STAGE PLANS: + Stage: Stage-1 +Map Reduce + Map Operator Tree: + TableScan +alias: src +Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: UDFToInteger(key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator +predicate: (((_col0 % 2) = 0) and (_col0 < 10)) (type: boolean) +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +Select Operator + expressions: _col0 (type: int), reverse(_col1) (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator +compressed: false +Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE +table: +input format: org.apache.hadoop.mapred.SequenceFileInputFormat +output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: -TableScan - alias: src - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator -expressions: UDFToInteger(key) (type: int), value (type: string) -outputColumnNames: _col0, _col1 -Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE -ListSink +ListSink PREHOOK: query: select * from masking_test_n13 PREHOOK: type: QUERY @@ -41,506 +61,12 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@masking_test_n13 POSTHOOK: Input: default@src A masked pattern was here -238val_238 -86 val_86 -311val_311 -27 val_27 -165val_165 -409val_409 -255val_255 -278val_278 -98 val_98 -484val_484 -265val_265 -193val_193 -401val_401 -150val_150 -273val_273 -224val_224 -369val_369 -66 val_66 -128val_128 -213val_213 -146val_146 -406val_406 -429val_429 -374val_374 -152val_152 -469val_469 -145val_145 -495val_495 -37 val_37 -327val_327 -281val_281 -277val_277 -209val_209 -15 val_15 -82 val_82 -403val_403 -166val_166 -417val_417 -430val_430 -252val_252 -292val_292 -219val_219 -287val_287 -153val_153 -193val_193 -338val_338 -446val_446 -459val_459 -394val_394 -237val_237 -482val_482 -174val_174 -413val_413 -494val_494 -207val_207 -199val_199 -466val_466 -208val_208 -174val_174 -399val_399 -396val_396 -247val_247 -417val_417 -489val_489 -162val_162 -377val_377 -397val_397 -309val_309 -365val_365 -266val_266 -439val_439 -342val_342 -367val_367 -325val_325 -167val_167 -195val_195 -475val_475 -17 val_17 -113val_113 -155val_155 -203val_203 -339val_339 -0 val_0 -455val_455 -128val_128 -311val_311 -316val_316 -57 val_57 -302val_302 -205val_205 -149val_149 -438val_438 -345val_345 -129val_129 -170val_170 -20 val_20 -489val_489 -157val_157 -378val_378 -221val_221 -92 val_92 -111val_111 -47 val_47 -72 val_72 -4 val_4 -280val_280 -35 val_35 -427val_427 -277val_277 -208val_208 -356val_356 -399val_399 -169val_169 -382val_382 -498val_498 -125val_125 -386val_386 -437val_437 -469val_469 -192val_192 -286val_286 -187val_187 -176val_176 -54 val_54 -459val_459 -51
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/branch-3.1 7e46905c6 -> 7e649028c
HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers
(Teddy Choi, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e649028
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e649028
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e649028
Branch: refs/heads/branch-3.1
Commit: 7e649028c343c3e5491d5fc5b3d271cb897f3f21
Parents: 7e46905
Author: Teddy Choi
Authored: Thu Jul 12 06:13:40 2018 +0900
Committer: Teddy Choi
Committed: Thu Jul 12 06:26:00 2018 +0900
--
.../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 6 ++
.../apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java | 4
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7e649028/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index d177e3f..889bd58 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -147,6 +147,12 @@ public class RecordReaderImpl extends
org.apache.orc.impl.RecordReaderImpl
public boolean nextBatch(VectorizedRowBatch theirBatch) throws IOException {
// If the user hasn't been reading by row, use the fast path.
if (rowInBatch >= batch.size) {
+ if (batch.size > 0) {
+// the local batch has been consumed entirely, reset it
+batch.reset();
+ }
+ baseRow = super.getRowNumber();
+ rowInBatch = 0;
return super.nextBatch(theirBatch);
}
copyIntoBatch(theirBatch, batch, rowInBatch);
http://git-wip-us.apache.org/repos/asf/hive/blob/7e649028/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
index 2071d13..aa99e57 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
@@ -154,9 +154,13 @@ public class TestVectorizedORCReader {
VectorizedRowBatch batch = reader.getSchema().createRowBatchV2();
OrcStruct row = null;
+long lastRowNumber = -1;
// Check Vectorized ORC reader against ORC row reader
while (vrr.nextBatch(batch)) {
+ Assert.assertEquals(lastRowNumber + 1, vrr.getRowNumber());
for (int i = 0; i < batch.size; i++) {
+Assert.assertEquals(rr.getRowNumber(), vrr.getRowNumber()+i);
+lastRowNumber = rr.getRowNumber();
row = (OrcStruct) rr.next(row);
for (int j = 0; j < batch.cols.length; j++) {
Object a = (row.getFieldValue(j));
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/branch-3 89382aee7 -> c829f61cb
HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers
(Teddy Choi, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c829f61c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c829f61c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c829f61c
Branch: refs/heads/branch-3
Commit: c829f61cbcc191b81d9fd2fb715e56ab2e7125fd
Parents: 89382ae
Author: Teddy Choi
Authored: Thu Jul 12 06:13:40 2018 +0900
Committer: Teddy Choi
Committed: Thu Jul 12 06:15:56 2018 +0900
--
.../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 6 ++
.../apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java | 4
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/c829f61c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index d177e3f..889bd58 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -147,6 +147,12 @@ public class RecordReaderImpl extends
org.apache.orc.impl.RecordReaderImpl
public boolean nextBatch(VectorizedRowBatch theirBatch) throws IOException {
// If the user hasn't been reading by row, use the fast path.
if (rowInBatch >= batch.size) {
+ if (batch.size > 0) {
+// the local batch has been consumed entirely, reset it
+batch.reset();
+ }
+ baseRow = super.getRowNumber();
+ rowInBatch = 0;
return super.nextBatch(theirBatch);
}
copyIntoBatch(theirBatch, batch, rowInBatch);
http://git-wip-us.apache.org/repos/asf/hive/blob/c829f61c/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
index 2071d13..aa99e57 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
@@ -154,9 +154,13 @@ public class TestVectorizedORCReader {
VectorizedRowBatch batch = reader.getSchema().createRowBatchV2();
OrcStruct row = null;
+long lastRowNumber = -1;
// Check Vectorized ORC reader against ORC row reader
while (vrr.nextBatch(batch)) {
+ Assert.assertEquals(lastRowNumber + 1, vrr.getRowNumber());
for (int i = 0; i < batch.size; i++) {
+Assert.assertEquals(rr.getRowNumber(), vrr.getRowNumber()+i);
+lastRowNumber = rr.getRowNumber();
row = (OrcStruct) rr.next(row);
for (int j = 0; j < batch.cols.length; j++) {
Object a = (row.getFieldValue(j));
hive git commit: HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers (Teddy Choi, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 733c4f316 -> 4a0814b7f
HIVE-20076: ACID: Fix Synthetic ROW__ID generation for vectorized orc readers
(Teddy Choi, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a0814b7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a0814b7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a0814b7
Branch: refs/heads/master
Commit: 4a0814b7f2edccb98f028a1528fc45c31a0d286f
Parents: 733c4f3
Author: Teddy Choi
Authored: Thu Jul 12 06:13:40 2018 +0900
Committer: Teddy Choi
Committed: Thu Jul 12 06:13:40 2018 +0900
--
.../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 6 ++
.../apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java | 4
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/4a0814b7/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index d177e3f..889bd58 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -147,6 +147,12 @@ public class RecordReaderImpl extends
org.apache.orc.impl.RecordReaderImpl
public boolean nextBatch(VectorizedRowBatch theirBatch) throws IOException {
// If the user hasn't been reading by row, use the fast path.
if (rowInBatch >= batch.size) {
+ if (batch.size > 0) {
+// the local batch has been consumed entirely, reset it
+batch.reset();
+ }
+ baseRow = super.getRowNumber();
+ rowInBatch = 0;
return super.nextBatch(theirBatch);
}
copyIntoBatch(theirBatch, batch, rowInBatch);
http://git-wip-us.apache.org/repos/asf/hive/blob/4a0814b7/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
--
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
index 2071d13..aa99e57 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
@@ -154,9 +154,13 @@ public class TestVectorizedORCReader {
VectorizedRowBatch batch = reader.getSchema().createRowBatchV2();
OrcStruct row = null;
+long lastRowNumber = -1;
// Check Vectorized ORC reader against ORC row reader
while (vrr.nextBatch(batch)) {
+ Assert.assertEquals(lastRowNumber + 1, vrr.getRowNumber());
for (int i = 0; i < batch.size; i++) {
+Assert.assertEquals(rr.getRowNumber(), vrr.getRowNumber()+i);
+lastRowNumber = rr.getRowNumber();
row = (OrcStruct) rr.next(row);
for (int j = 0; j < batch.cols.length; j++) {
Object a = (row.getFieldValue(j));
[1/4] hive git commit: HIVE-20106 Backport HIVE-20060 (HiveSchemaTool and MetastoreSchemaTool refactor) to branch-3 (Alan Gates, original patch reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/branch-3 e20592cb2 -> 89382aee7
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
new file mode 100644
index 000..cd1d57b
--- /dev/null
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInfo.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import
org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
+
+/**
+ * Print Hive version and schema version.
+ */
+class SchemaToolTaskInfo extends SchemaToolTask {
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+// do nothing
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+String hiveVersion =
schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+MetaStoreConnectionInfo connectionInfo =
schemaTool.getConnectionInfo(true);
+String dbVersion =
schemaTool.getMetaStoreSchemaInfo().getMetaStoreSchemaVersion(connectionInfo);
+
+System.out.println("Hive distribution version:\t " + hiveVersion);
+System.out.println("Metastore schema version:\t " + dbVersion);
+
+schemaTool.assertCompatibleVersion(hiveVersion, dbVersion);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
new file mode 100644
index 000..e3fa495
--- /dev/null
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SchemaToolTaskInit.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+/**
+ * Initialize the metastore schema.
+ */
+class SchemaToolTaskInit extends SchemaToolTask {
+ private boolean validate = true;
+ private String toVersion;
+
+ @Override
+ void setCommandLineArguments(SchemaToolCommandLine cl) {
+if (cl.hasOption("initSchemaTo")) {
+ this.toVersion = cl.getOptionValue("initSchemaTo");
+ this.validate = false;
+}
+ }
+
+ private void ensureToVersion() throws HiveMetaException {
+if (toVersion != null) {
+ return;
+}
+
+// If null then current hive version is used
+toVersion = schemaTool.getMetaStoreSchemaInfo().getHiveSchemaVersion();
+System.out.println("Initializing the schema to: " + toVersion);
+ }
+
+ @Override
+ void execute() throws HiveMetaException {
+ensureToVersion();
+
+schemaTool.testConnectionToMetastore();
+System.out.println("Starting metastore schema initialization to " +
toVersion);
+
+
[4/4] hive git commit: HIVE-20106 Backport HIVE-20060 (HiveSchemaTool and MetastoreSchemaTool refactor) to branch-3 (Alan Gates, original patch reviewed by Daniel Dai)
HIVE-20106 Backport HIVE-20060 (HiveSchemaTool and MetastoreSchemaTool
refactor) to branch-3 (Alan Gates, original patch reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/89382aee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/89382aee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/89382aee
Branch: refs/heads/branch-3
Commit: 89382aee74e7f3be5cb4060dc0cd9af42c46d3ed
Parents: e20592c
Author: Alan Gates
Authored: Wed Jul 11 14:02:13 2018 -0700
Committer: Alan Gates
Committed: Wed Jul 11 14:02:13 2018 -0700
--
.../hive/beeline/schematool/HiveSchemaTool.java | 338 +
.../schematool/HiveSchemaToolCommandLine.java | 286
.../beeline/schematool/HiveSchemaToolTask.java | 32 -
.../HiveSchemaToolTaskAlterCatalog.java | 90 --
.../HiveSchemaToolTaskCreateCatalog.java| 132 --
.../schematool/HiveSchemaToolTaskInfo.java | 43 -
.../schematool/HiveSchemaToolTaskInit.java | 73 -
.../HiveSchemaToolTaskMoveDatabase.java | 96 --
.../schematool/HiveSchemaToolTaskMoveTable.java | 142 --
.../schematool/HiveSchemaToolTaskUpgrade.java | 116 --
.../schematool/HiveSchemaToolTaskValidate.java | 631 -
.../beeline/schematool/TestHiveSchemaTool.java |6 +-
.../tools/TestSchemaToolCatalogOps.java | 479 +++
.../hive/beeline/schematool/TestSchemaTool.java | 550 +---
.../schematool/TestSchemaToolCatalogOps.java| 478 ---
.../metastore/tools/MetastoreSchemaTool.java| 1282 +++---
.../metastore/tools/SchemaToolCommandLine.java | 308 +
.../hive/metastore/tools/SchemaToolTask.java| 32 +
.../tools/SchemaToolTaskAlterCatalog.java | 90 ++
.../tools/SchemaToolTaskCreateCatalog.java | 132 ++
.../tools/SchemaToolTaskCreateUser.java | 115 ++
.../metastore/tools/SchemaToolTaskInfo.java | 43 +
.../metastore/tools/SchemaToolTaskInit.java | 73 +
.../tools/SchemaToolTaskMoveDatabase.java | 96 ++
.../tools/SchemaToolTaskMoveTable.java | 142 ++
.../metastore/tools/SchemaToolTaskUpgrade.java | 116 ++
.../metastore/tools/SchemaToolTaskValidate.java | 630 +
.../hive/metastore/dbinstall/DbInstallBase.java | 25 +-
.../tools/TestSchemaToolForMetastore.java | 277 ++--
29 files changed, 2708 insertions(+), 4145 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
--
diff --git
a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
index 9c3f30b..69514e5 100644
--- a/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
+++ b/beeline/src/java/org/apache/hive/beeline/schematool/HiveSchemaTool.java
@@ -17,207 +17,50 @@
*/
package org.apache.hive.beeline.schematool;
-import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaException;
-import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
-import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper;
import
org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
import
org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParser;
-import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.hive.metastore.tools.MetastoreSchemaTool;
import org.apache.hive.beeline.BeeLine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
-import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
-import java.net.URI;
-import java.sql.Connection;
-import java.sql.SQLException;
-public class HiveSchemaTool {
+public class HiveSchemaTool extends MetastoreSchemaTool {
private static final Logger LOG =
LoggerFactory.getLogger(HiveSchemaTool.class.getName());
- private final HiveConf hiveConf;
- private final String dbType;
- private final
[3/4] hive git commit: HIVE-20106 Backport HIVE-20060 (HiveSchemaTool and MetastoreSchemaTool refactor) to branch-3 (Alan Gates, original patch reviewed by Daniel Dai)
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
--
diff --git
a/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
b/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
index 8b477bd..8514dc8 100644
---
a/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
+++
b/beeline/src/test/org/apache/hive/beeline/schematool/TestHiveSchemaTool.java
@@ -43,13 +43,13 @@ import static
org.powermock.api.mockito.PowerMockito.verifyStatic;
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*")
-@PrepareForTest({ HiveSchemaHelper.class, HiveSchemaTool.CommandBuilder.class
})
+@PrepareForTest({ HiveSchemaHelper.class,
HiveSchemaTool.HiveSchemaToolCommandBuilder.class })
public class TestHiveSchemaTool {
String scriptFile = System.getProperty("java.io.tmpdir") + File.separator +
"someScript.sql";
@Mock
private HiveConf hiveConf;
- private HiveSchemaTool.CommandBuilder builder;
+ private HiveSchemaTool.HiveSchemaToolCommandBuilder builder;
private String pasword = "reallySimplePassword";
@Before
@@ -66,7 +66,7 @@ public class TestHiveSchemaTool {
if (!file.exists()) {
file.createNewFile();
}
-builder = new HiveSchemaTool.CommandBuilder(hiveConf, null, null,
"testUser", pasword, scriptFile);
+builder = new HiveSchemaTool.HiveSchemaToolCommandBuilder(hiveConf, null,
null, "testUser", pasword, scriptFile);
}
@After
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
new file mode 100644
index 000..42eb979
--- /dev/null
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/TestSchemaToolCatalogOps.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.text.StrTokenizer;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Catalog;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.FunctionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.thrift.TException;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+
+public class TestSchemaToolCatalogOps {
+ private static MetastoreSchemaTool schemaTool;
+ private static HiveConf conf;
+ private IMetaStoreClient client;
+ private static String testMetastoreDB;
+ private static String argsBase;
+
+ @BeforeClass
+ public static void initDb() throws HiveMetaException,
[2/4] hive git commit: HIVE-20106 Backport HIVE-20060 (HiveSchemaTool and MetastoreSchemaTool refactor) to branch-3 (Alan Gates, original patch reviewed by Daniel Dai)
http://git-wip-us.apache.org/repos/asf/hive/blob/89382aee/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
--
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
index 7c8087f..c2018f4 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/MetastoreSchemaTool.java
@@ -18,25 +18,15 @@
package org.apache.hadoop.hive.metastore.tools;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.output.NullOutputStream;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.HiveMetaException;
import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo;
import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory;
-import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import
org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.MetaStoreConnectionInfo;
@@ -44,55 +34,44 @@ import
org.apache.hadoop.hive.metastore.tools.HiveSchemaHelper.NestedScriptParse
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableMap;
import sqlline.SqlLine;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.URI;
import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
public class MetastoreSchemaTool {
private static final Logger LOG =
LoggerFactory.getLogger(MetastoreSchemaTool.class);
private static final String PASSWD_MASK = "[passwd stripped]";
- @VisibleForTesting
- public static String homeDir;
-
- private String userName = null;
- private String passWord = null;
- private boolean dryRun = false;
- private boolean verbose = false;
- private String dbOpts = null;
- private String url = null;
- private String driver = null;
- private URI[] validationServers = null; // The list of servers the
database/partition/table can locate on
- private String hiveUser; // Hive username, for use when creating the user,
not for connecting
- private String hivePasswd; // Hive password, for use when creating the user,
not for connecting
- private String hiveDb; // Hive database, for use when creating the user, not
for connecting
- private final Configuration conf;
- private final String dbType;
- private final IMetaStoreSchemaInfo metaStoreSchemaInfo;
- private boolean needsQuotedIdentifier;
+ protected Configuration conf;
+
+ protected String dbOpts = null;
+ protected String dbType;
+ protected String driver = null;
+ protected boolean dryRun = false;
+ protected String hiveDb; // Hive database, for use when creating the user,
not for connecting
+ protected String hivePasswd; // Hive password, for use when creating the
user, not for connecting
+ protected String hiveUser; // Hive username, for use when creating the user,
not for connecting
+ protected String metaDbType;
+ protected IMetaStoreSchemaInfo metaStoreSchemaInfo;
+ protected boolean needsQuotedIdentifier;
+ protected String quoteCharacter;
+ protected String passWord = null;
+ protected String url = null;
+ protected String userName = null;
+ protected URI[] validationServers = null; // The list of servers the
database/partition/table can locate on
+ protected boolean verbose = false;
+ protected SchemaToolCommandLine cmdLine;
+
+ private static String homeDir;
private static String findHomeDir() {
// If METASTORE_HOME is set, use it, else use
hive git commit: HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache (Eugene Koifman, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 385a26ad8 -> 733c4f316
HIVE-19387: Truncate table for Acid tables conflicts with ResultSet cache
(Eugene Koifman, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/733c4f31
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/733c4f31
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/733c4f31
Branch: refs/heads/master
Commit: 733c4f3169142d426995f8e12961c209688d29d0
Parents: 385a26a
Author: Eugene Koifman
Authored: Wed Jul 11 14:02:31 2018 -0700
Committer: Eugene Koifman
Committed: Wed Jul 11 14:02:31 2018 -0700
--
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/parse/DDLSemanticAnalyzer.java | 11 +-
.../hadoop/hive/ql/plan/AlterTableDesc.java | 5 +
.../hadoop/hive/ql/plan/TruncateTableDesc.java | 33 +-
.../clientpositive/results_cache_truncate.q | 61 +++
.../llap/results_cache_truncate.q.out | 470 +++
6 files changed, 579 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 9e012ce..72dd144 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -610,6 +610,7 @@ minillaplocal.query.files=\
results_cache_quoted_identifiers.q,\
results_cache_temptable.q,\
results_cache_transactional.q,\
+ results_cache_truncate.q,\
results_cache_with_masking.q,\
sample10.q,\
sample10_mm.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 9ad4689..b6825ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1478,7 +1478,13 @@ public class DDLSemanticAnalyzer extends
BaseSemanticAnalyzer {
}
}
-TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName,
partSpec, null);
+TruncateTableDesc truncateTblDesc = new TruncateTableDesc(tableName,
partSpec, null, table);
+if(truncateTblDesc.mayNeedWriteId()) {
+ if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: "
+ this.ddlDescWithWriteId);
+ }
+ this.ddlDescWithWriteId = truncateTblDesc;
+}
DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), truncateTblDesc);
Task truncateTask = TaskFactory.get(ddlWork);
@@ -1757,6 +1763,9 @@ public class DDLSemanticAnalyzer extends
BaseSemanticAnalyzer {
DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc);
if (isPotentialMmSwitch) {
+ if(this.ddlDescWithWriteId != null) {
+throw new IllegalStateException("ddlDescWithWriteId is already set: "
+ this.ddlDescWithWriteId);
+ }
this.ddlDescWithWriteId = alterTblDesc;
ddlWork.setNeedLock(true); // Hmm... why don't many other operations
here need locks?
}
http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
index 0b04c0c..ec04a01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
@@ -967,4 +967,9 @@ public class AlterTableDesc extends DDLDesc implements
Serializable, DDLDesc.DDL
public Long getWriteId() {
return this.writeId;
}
+
+ @Override
+ public String toString() {
+return this.getClass().getSimpleName() + " for " + getFullTableName();
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/733c4f31/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
index ed68f1a..8c3d852 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TruncateTableDesc.java
+++
hive git commit: HIVE-19984 Backport HIVE-15976 (CURRENT_SCHEMA and CURRENT_CATALOG) to branch-3 (Lazlo Bodor via Alan Gates)
Repository: hive
Updated Branches:
refs/heads/branch-3 2df5165fe -> e20592cb2
HIVE-19984 Backport HIVE-15976 (CURRENT_SCHEMA and CURRENT_CATALOG) to branch-3
(Lazlo Bodor via Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e20592cb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e20592cb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e20592cb
Branch: refs/heads/branch-3
Commit: e20592cb27c0556b61771e73997f9c1855bff575
Parents: 2df5165
Author: Alan Gates
Authored: Wed Jul 11 13:36:20 2018 -0700
Committer: Alan Gates
Committed: Wed Jul 11 13:36:20 2018 -0700
--
.../hadoop/hive/ql/exec/FunctionRegistry.java | 4 +-
.../udf/generic/GenericUDFCurrentCatalog.java | 52 +++
.../udf/generic/GenericUDFCurrentDatabase.java | 53 +++
.../ql/udf/generic/GenericUDFCurrentSchema.java | 37 +++
.../hive/ql/udf/generic/UDFCurrentDB.java | 68
.../clientpositive/current_catalog_and_schema.q | 10 +++
.../current_catalog_and_schema.q.out| 54
.../results/clientpositive/show_functions.q.out | 4 ++
8 files changed, 213 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/e20592cb/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 9ddfc06..10517ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -352,7 +352,9 @@ public final class FunctionRegistry {
system.registerGenericUDF("grouping", GenericUDFGrouping.class);
-system.registerGenericUDF("current_database", UDFCurrentDB.class);
+system.registerGenericUDF("current_database",
GenericUDFCurrentDatabase.class);
+system.registerGenericUDF("current_schema", GenericUDFCurrentSchema.class);
+system.registerGenericUDF("current_catalog",
GenericUDFCurrentCatalog.class);
system.registerGenericUDF("current_date", GenericUDFCurrentDate.class);
system.registerGenericUDF("current_timestamp",
GenericUDFCurrentTimestamp.class);
system.registerGenericUDF("current_user", GenericUDFCurrentUser.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/e20592cb/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentCatalog.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentCatalog.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentCatalog.java
new file mode 100644
index 000..e2684c8
--- /dev/null
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentCatalog.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+
+// This function is not a deterministic function, but a runtime constant.
+// The return value is constant within a query but can be different between
queries.
+@UDFType(deterministic = false, runtimeConstant = true)
+@Description(name = "current_catalog", value = "_FUNC_() - returns currently
used catalog name")
+@NDV(maxNdv = 1)
+public class GenericUDFCurrentCatalog extends GenericUDF {
+ @Override
+ public
hive git commit: HIVE-20093: LlapOutputFomatService: Use ArrowBuf with Netty for Accounting (Eric Wohlstadter, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 40635f7e3 -> 385a26ad8
HIVE-20093: LlapOutputFomatService: Use ArrowBuf with Netty for Accounting
(Eric Wohlstadter, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/385a26ad
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/385a26ad
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/385a26ad
Branch: refs/heads/master
Commit: 385a26ad89a29492ff55bcba32ec104af54bf139
Parents: 40635f7
Author: Jason Dere
Authored: Wed Jul 11 10:47:01 2018 -0700
Committer: Jason Dere
Committed: Wed Jul 11 10:47:01 2018 -0700
--
.../apache/hadoop/hive/llap/LlapOutputFormatService.java | 4 +++-
.../hadoop/hive/llap/WritableByteChannelAdapter.java | 11 ---
2 files changed, 11 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/385a26ad/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
index c71c637..996f8b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapOutputFormatService.java
@@ -199,10 +199,12 @@ public class LlapOutputFormatService {
int maxPendingWrites = HiveConf.getIntVar(conf,
HiveConf.ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES);
boolean useArrow = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.LLAP_OUTPUT_FORMAT_ARROW);
+ long allocatorMax = HiveConf.getLongVar(conf,
+ HiveConf.ConfVars.HIVE_ARROW_ROOT_ALLOCATOR_LIMIT);
@SuppressWarnings("rawtypes")
RecordWriter writer = null;
if(useArrow) {
-writer = new LlapArrowRecordWriter(new WritableByteChannelAdapter(ctx,
maxPendingWrites, id));
+writer = new LlapArrowRecordWriter(new WritableByteChannelAdapter(ctx,
maxPendingWrites, id, allocatorMax));
} else {
writer = new LlapRecordWriter(id,
new ChunkedOutputStream(
http://git-wip-us.apache.org/repos/asf/hive/blob/385a26ad/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
b/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
index 57da1d9..753da22 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.hive.llap;
-import io.netty.buffer.Unpooled;
+import org.apache.hadoop.hive.ql.io.arrow.RootAllocatorFactory;
+import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import java.io.IOException;
@@ -48,6 +49,7 @@ public class WritableByteChannelAdapter implements
WritableByteChannel {
private final Semaphore writeResources;
private boolean closed = false;
private final String id;
+ private long allocatorMax;
private ChannelFutureListener writeListener = new ChannelFutureListener() {
@Override
@@ -75,11 +77,12 @@ public class WritableByteChannelAdapter implements
WritableByteChannel {
}
};
- public WritableByteChannelAdapter(ChannelHandlerContext chc, int
maxPendingWrites, String id) {
+ public WritableByteChannelAdapter(ChannelHandlerContext chc, int
maxPendingWrites, String id, long allocatorMax) {
this.chc = chc;
this.maxPendingWrites = maxPendingWrites;
this.writeResources = new Semaphore(maxPendingWrites);
this.id = id;
+this.allocatorMax = allocatorMax;
}
@Override
@@ -87,7 +90,9 @@ public class WritableByteChannelAdapter implements
WritableByteChannel {
int size = src.remaining();
//Down the semaphore or block until available
takeWriteResources(1);
-chc.writeAndFlush(Unpooled.wrappedBuffer(src)).addListener(writeListener);
+ByteBuf buf =
RootAllocatorFactory.INSTANCE.getOrCreateRootAllocator(allocatorMax).buffer(size);
+buf.writeBytes(src);
+chc.writeAndFlush(buf).addListener(writeListener);
return size;
}
hive git commit: HIVE-20135: Fix incompatible change in TimestampColumnVector to default to UTC (Jesus Camacho Rodriguez, reviewed by Owen O'Malley)
Repository: hive
Updated Branches:
refs/heads/branch-3.1 411ce3504 -> 7e46905c6
HIVE-20135: Fix incompatible change in TimestampColumnVector to default to UTC
(Jesus Camacho Rodriguez, reviewed by Owen O'Malley)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7e46905c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7e46905c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7e46905c
Branch: refs/heads/branch-3.1
Commit: 7e46905c6571ef3e8bec1f17811037baa55aacc9
Parents: 411ce35
Author: Jesus Camacho Rodriguez
Authored: Wed Jul 11 07:58:53 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Jul 11 09:08:12 2018 -0700
--
.../apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java | 2 +-
.../apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/7e46905c/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
--
diff --git
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index f11a319..3b5f3ba 100644
---
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -81,7 +81,7 @@ public class TimestampColumnVector extends ColumnVector {
scratchWritable = null; // Allocated by caller.
-isUTC = true;
+isUTC = false;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/7e46905c/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
--
diff --git
a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
index fbd0c06..d1a546f 100644
---
a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
+++
b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
@@ -105,6 +105,7 @@ public class TestStructColumnVector {
VectorizedRowBatch batch = new VectorizedRowBatch(2);
LongColumnVector x1 = new LongColumnVector();
TimestampColumnVector x2 = new TimestampColumnVector();
+x2.setIsUTC(true);
StructColumnVector x = new StructColumnVector(1024, x1, x2);
BytesColumnVector y = new BytesColumnVector();
batch.cols[0] = x;
hive git commit: HIVE-20135: Fix incompatible change in TimestampColumnVector to default to UTC (Jesus Camacho Rodriguez, reviewed by Owen O'Malley)
Repository: hive
Updated Branches:
refs/heads/branch-3 6c55afb04 -> 2df5165fe
HIVE-20135: Fix incompatible change in TimestampColumnVector to default to UTC
(Jesus Camacho Rodriguez, reviewed by Owen O'Malley)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2df5165f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2df5165f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2df5165f
Branch: refs/heads/branch-3
Commit: 2df5165fe932fc96c135dd7c4c3dc56627efa53b
Parents: 6c55afb
Author: Jesus Camacho Rodriguez
Authored: Wed Jul 11 07:58:53 2018 -0700
Committer: Jesus Camacho Rodriguez
Committed: Wed Jul 11 09:07:47 2018 -0700
--
.../apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java | 2 +-
.../apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/2df5165f/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
--
diff --git
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index f11a319..3b5f3ba 100644
---
a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++
b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -81,7 +81,7 @@ public class TimestampColumnVector extends ColumnVector {
scratchWritable = null; // Allocated by caller.
-isUTC = true;
+isUTC = false;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/2df5165f/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
--
diff --git
a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
index fbd0c06..d1a546f 100644
---
a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
+++
b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
@@ -105,6 +105,7 @@ public class TestStructColumnVector {
VectorizedRowBatch batch = new VectorizedRowBatch(2);
LongColumnVector x1 = new LongColumnVector();
TimestampColumnVector x2 = new TimestampColumnVector();
+x2.setIsUTC(true);
StructColumnVector x = new StructColumnVector(1024, x1, x2);
BytesColumnVector y = new BytesColumnVector();
batch.cols[0] = x;
