This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 71985c2 HIVE-21678 : CTAS creating a Full ACID partitioned table
fails because of no writeId. (Ashutosh Bapat reviewed by Mahesh Kumar Behera,
Sankar Hariappan)
71985c2 is described below
commit 71985c23b9c01a764d88274f21f9ce014e5e1a7e
Author: Ashutosh Bapat <aba...@cloudera.com>
AuthorDate: Wed May 8 14:34:01 2019 +0530
HIVE-21678 : CTAS creating a Full ACID partitioned table fails because of
no writeId. (Ashutosh Bapat reviewed by Mahesh Kumar Behera, Sankar Hariappan)
Signed-off-by: Mahesh Kumar Behera <mah...@apache.org>
---
.../hadoop/hive/ql/parse/ReplicationTestUtils.java | 4 +--
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +++--
ql/src/test/queries/clientpositive/ctas.q | 18 +++++-----
ql/src/test/results/clientpositive/ctas.q.out | 40 ++++++++++++++++++++++
ql/src/test/results/clientpositive/llap/ctas.q.out | 40 ++++++++++++++++++++++
.../test/results/clientpositive/spark/ctas.q.out | 40 ++++++++++++++++++++++
6 files changed, 137 insertions(+), 13 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
index bde6570..4caa884 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java
@@ -136,10 +136,10 @@ public class ReplicationTestUtils {
String tableNameCTAS = tableName + "_CTAS";
String tableNameCTASMM = tableName + "_CTASMM";
- /*insertRecords(primary, primaryDbName, primaryDbNameExtra,
+ insertRecords(primary, primaryDbName, primaryDbNameExtra,
tableName, tableNameCTAS, false,
OperationType.REPL_TEST_ACID_CTAS);
selectStmtList.add("select key from " + tableNameCTAS + " order by key");
- expectedValues.add(new String[]{"1", "2", "3", "4", "5"});*/
+ expectedValues.add(new String[]{"1", "2", "3", "4", "5"});
insertRecords(primary, primaryDbName, primaryDbNameExtra,
tableNameMM, tableNameCTASMM, true,
OperationType.REPL_TEST_ACID_CTAS);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index f436611..a3da075 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7477,8 +7477,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
fileSinkColInfos = new ArrayList<>();
destTableIsTemporary = tblDesc.isTemporary();
destTableIsMaterialization = tblDesc.isMaterialization();
- if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
- isMmTable = isMmCtas = true;
+ if (AcidUtils.isTablePropertyTransactional(tblDesc.getTblProps())) {
try {
if (ctx.getExplainConfig() != null) {
writeId = 0L; // For explain plan, txn won't be opened and
doesn't make sense to allocate write id
@@ -7498,7 +7497,10 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
} catch (LockException ex) {
throw new SemanticException("Failed to allocate write Id", ex);
}
- tblDesc.setInitialMmWriteId(writeId);
+ if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
+ isMmTable = isMmCtas = true;
+ tblDesc.setInitialMmWriteId(writeId);
+ }
}
} else if (viewDesc != null) {
fieldSchemas = new ArrayList<>();
diff --git a/ql/src/test/queries/clientpositive/ctas.q
b/ql/src/test/queries/clientpositive/ctas.q
index c4fdda1..cd0bf5a 100644
--- a/ql/src/test/queries/clientpositive/ctas.q
+++ b/ql/src/test/queries/clientpositive/ctas.q
@@ -61,11 +61,13 @@ create table nzhang_ctas6 (key string, `to` string);
insert overwrite table nzhang_ctas6 select key, value from src tablesample (10
rows);
create table nzhang_ctas7 as select key, `to` from nzhang_ctas6;
-
-
-
-
-
-
-
-
+-- ACID CTAS
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.stats.autogather=false;
+
+create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5;
+select k, value from acid_ctas_part;
diff --git a/ql/src/test/results/clientpositive/ctas.q.out
b/ql/src/test/results/clientpositive/ctas.q.out
index 1074730..d181df0 100644
--- a/ql/src/test/results/clientpositive/ctas.q.out
+++ b/ql/src/test/results/clientpositive/ctas.q.out
@@ -1075,3 +1075,43 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@nzhang_ctas7
POSTHOOK: Lineage: nzhang_ctas7.key SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: nzhang_ctas7.to SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:to, type:string, comment:null), ]
+PREHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_ctas_part
+PREHOOK: Output: default@acid_ctas_part
+POSTHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_ctas_part
+POSTHOOK: Output: default@acid_ctas_part@k=0
+POSTHOOK: Output: default@acid_ctas_part@k=10
+POSTHOOK: Output: default@acid_ctas_part@k=100
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=0).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=100).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=10).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select k, value from acid_ctas_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_ctas_part
+PREHOOK: Input: default@acid_ctas_part@k=0
+PREHOOK: Input: default@acid_ctas_part@k=10
+PREHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+POSTHOOK: query: select k, value from acid_ctas_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_ctas_part
+POSTHOOK: Input: default@acid_ctas_part@k=0
+POSTHOOK: Input: default@acid_ctas_part@k=10
+POSTHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out
b/ql/src/test/results/clientpositive/llap/ctas.q.out
index 38851b1..bd57695 100644
--- a/ql/src/test/results/clientpositive/llap/ctas.q.out
+++ b/ql/src/test/results/clientpositive/llap/ctas.q.out
@@ -1040,3 +1040,43 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@nzhang_ctas7
POSTHOOK: Lineage: nzhang_ctas7.key SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: nzhang_ctas7.to SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:to, type:string, comment:null), ]
+PREHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_ctas_part
+PREHOOK: Output: default@acid_ctas_part
+POSTHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_ctas_part
+POSTHOOK: Output: default@acid_ctas_part@k=0
+POSTHOOK: Output: default@acid_ctas_part@k=10
+POSTHOOK: Output: default@acid_ctas_part@k=100
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=0).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=100).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=10).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select k, value from acid_ctas_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_ctas_part
+PREHOOK: Input: default@acid_ctas_part@k=0
+PREHOOK: Input: default@acid_ctas_part@k=10
+PREHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+POSTHOOK: query: select k, value from acid_ctas_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_ctas_part
+POSTHOOK: Input: default@acid_ctas_part@k=0
+POSTHOOK: Input: default@acid_ctas_part@k=10
+POSTHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out
b/ql/src/test/results/clientpositive/spark/ctas.q.out
index b1fa6ec..4a01dd4 100644
--- a/ql/src/test/results/clientpositive/spark/ctas.q.out
+++ b/ql/src/test/results/clientpositive/spark/ctas.q.out
@@ -840,3 +840,43 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@nzhang_ctas7
POSTHOOK: Lineage: nzhang_ctas7.key SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:key, type:string, comment:null), ]
POSTHOOK: Lineage: nzhang_ctas7.to SIMPLE
[(nzhang_ctas6)nzhang_ctas6.FieldSchema(name:to, type:string, comment:null), ]
+PREHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@acid_ctas_part
+PREHOOK: Output: default@acid_ctas_part
+POSTHOOK: query: create table acid_ctas_part partitioned by (k)
+ stored as orc TBLPROPERTIES ('transactional'='true')
+ as select key k, value from src order by k limit 5
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@acid_ctas_part
+POSTHOOK: Output: default@acid_ctas_part@k=0
+POSTHOOK: Output: default@acid_ctas_part@k=10
+POSTHOOK: Output: default@acid_ctas_part@k=100
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=0).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=100).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: acid_ctas_part PARTITION(k=10).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select k, value from acid_ctas_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@acid_ctas_part
+PREHOOK: Input: default@acid_ctas_part@k=0
+PREHOOK: Input: default@acid_ctas_part@k=10
+PREHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+POSTHOOK: query: select k, value from acid_ctas_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@acid_ctas_part
+POSTHOOK: Input: default@acid_ctas_part@k=0
+POSTHOOK: Input: default@acid_ctas_part@k=10
+POSTHOOK: Input: default@acid_ctas_part@k=100
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
