HIVE-19280 : Invalid error messages for UPDATE/DELETE on insert-only transactional tables (Steve Yeom, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dd343d5f Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dd343d5f Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dd343d5f Branch: refs/heads/storage-branch-2.6 Commit: dd343d5f0760c8ffe25aabd2fbf4c36d4081da97 Parents: 6607f8f Author: sergey <ser...@apache.org> Authored: Wed Apr 25 11:19:18 2018 -0700 Committer: sergey <ser...@apache.org> Committed: Wed Apr 25 11:24:44 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 16 +++-- ql/src/test/queries/clientnegative/mm_delete.q | 17 +++++ ql/src/test/queries/clientnegative/mm_update.q | 15 +++++ .../test/results/clientnegative/mm_delete.q.out | 68 ++++++++++++++++++++ .../test/results/clientnegative/mm_update.q.out | 58 +++++++++++++++++ 6 files changed, 170 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index fde16f8..7d33fa3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -460,6 +460,8 @@ public enum ErrorMsg { LOAD_DATA_ACID_FILE(10413, "\"{0}\" was created created by Acid write - it cannot be loaded into anther Acid table", true), + ACID_OP_ON_INSERTONLYTRAN_TABLE(10414, "Attempt to do update or delete on table {0} that is " + + "insert-only transactional", true), //========================== 20000 range starts here ========================// http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index a00f927..1dccf96 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -2236,12 +2236,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { "Inconsistent data structure detected: we are writing to " + ts.tableHandle + " in " + name + " but it's not in isInsertIntoTable() or getInsertOverwriteTables()"; // Disallow update and delete on non-acid tables - boolean isAcid = AcidUtils.isFullAcidTable(ts.tableHandle); - if ((updating(name) || deleting(name)) && !isAcid) { - // Whether we are using an acid compliant transaction manager has already been caught in - // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid - // here, it means the table itself doesn't support it. - throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName); + boolean isFullAcid = AcidUtils.isFullAcidTable(ts.tableHandle); + if ((updating(name) || deleting(name)) && !isFullAcid) { + if (!AcidUtils.isInsertOnlyTable(ts.tableHandle)) { + // Whether we are using an acid compliant transaction manager has already been caught in + // UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid + // here, it means the table itself doesn't support it. + throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, ts.tableName); + } else { + throw new SemanticException(ErrorMsg.ACID_OP_ON_INSERTONLYTRAN_TABLE, ts.tableName); + } } // TableSpec ts is got from the query (user specified), // which means the user didn't specify partitions in their query, http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/test/queries/clientnegative/mm_delete.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/mm_delete.q b/ql/src/test/queries/clientnegative/mm_delete.q new file mode 100644 index 0000000..f0e92dc --- /dev/null +++ b/ql/src/test/queries/clientnegative/mm_delete.q @@ -0,0 +1,17 @@ +--! qt:dataset:srcpart +set hive.mapred.mode=nonstrict; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.explain.user=false; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table if exists mm_srcpart; +CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only'); +insert into mm_srcpart PARTITION (ds, hr) select * from srcpart; + +select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); + +insert into mm_srcpart PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003'); + +delete from mm_srcpart where key in( '1001', '213', '43'); + http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/test/queries/clientnegative/mm_update.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/mm_update.q b/ql/src/test/queries/clientnegative/mm_update.q new file mode 100644 index 0000000..2723027 --- /dev/null +++ b/ql/src/test/queries/clientnegative/mm_update.q @@ -0,0 +1,15 @@ +--! qt:dataset:srcpart +set hive.mapred.mode=nonstrict; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.explain.user=false; +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; + +drop table if exists mm_srcpart; +CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only'); +insert into mm_srcpart PARTITION (ds, hr) select * from srcpart; + +select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer); + +update mm_srcpart set value = concat(value, 'updated') where cast(key as integer) in(413,43) and hr='11'; + http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/test/results/clientnegative/mm_delete.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/mm_delete.q.out b/ql/src/test/results/clientnegative/mm_delete.q.out new file mode 100644 index 0000000..ed7bafb --- /dev/null +++ b/ql/src/test/results/clientnegative/mm_delete.q.out @@ -0,0 +1,68 @@ +PREHOOK: query: drop table if exists mm_srcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists mm_srcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mm_srcpart +POSTHOOK: query: CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mm_srcpart +PREHOOK: query: insert into mm_srcpart PARTITION (ds, hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@mm_srcpart +POSTHOOK: query: insert into mm_srcpart PARTITION (ds, hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer) +PREHOOK: type: QUERY +PREHOOK: Input: default@mm_srcpart +PREHOOK: Input: default@mm_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@mm_srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mm_srcpart +POSTHOOK: Input: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@mm_srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +2008-04-08 11 43 val_43 +2008-04-08 11 413 val_413 +2008-04-08 11 413 val_413 +2008-04-09 11 43 val_43 +2008-04-09 11 413 val_413 +2008-04-09 11 413 val_413 +PREHOOK: query: insert into mm_srcpart PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert into mm_srcpart PARTITION (ds='2008-04-08', hr=='11') values ('1001','val1001'),('1002','val1002'),('1003','val1003') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).key SCRIPT [] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).value SCRIPT [] +FAILED: SemanticException [Error 10414]: Attempt to do update or delete on table default.mm_srcpart that is insert-only transactional http://git-wip-us.apache.org/repos/asf/hive/blob/dd343d5f/ql/src/test/results/clientnegative/mm_update.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/mm_update.q.out b/ql/src/test/results/clientnegative/mm_update.q.out new file mode 100644 index 0000000..946ffd1 --- /dev/null +++ b/ql/src/test/results/clientnegative/mm_update.q.out @@ -0,0 +1,58 @@ +PREHOOK: query: drop table if exists mm_srcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists mm_srcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@mm_srcpart +POSTHOOK: query: CREATE TABLE mm_srcpart (key STRING, value STRING) PARTITIONED BY (ds STRING, hr STRING) stored as ORC TBLPROPERTIES ('transactional'='true', 'transactional_properties'='insert_only') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@mm_srcpart +PREHOOK: query: insert into mm_srcpart PARTITION (ds, hr) select * from srcpart +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@mm_srcpart +POSTHOOK: query: insert into mm_srcpart PARTITION (ds, hr) select * from srcpart +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@mm_srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: mm_srcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer) +PREHOOK: type: QUERY +PREHOOK: Input: default@mm_srcpart +PREHOOK: Input: default@mm_srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@mm_srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +POSTHOOK: query: select ds, hr, key, value from mm_srcpart where cast(key as integer) in(413,43) and hr='11' order by ds, hr, cast(key as integer) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@mm_srcpart +POSTHOOK: Input: default@mm_srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@mm_srcpart@ds=2008-04-09/hr=11 +#### A masked pattern was here #### +2008-04-08 11 43 val_43 +2008-04-08 11 413 val_413 +2008-04-08 11 413 val_413 +2008-04-09 11 43 val_43 +2008-04-09 11 413 val_413 +2008-04-09 11 413 val_413 +FAILED: SemanticException [Error 10414]: Attempt to do update or delete on table default.mm_srcpart that is insert-only transactional