hive git commit: HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene Koifman, reviewed by Sergey Shelukhin)

2018-05-04 Thread ekoifman 
Repository: hive
Updated Branches:
  refs/heads/master 552dbaf33 -> 8158f8848


HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8158f884
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8158f884
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8158f884

Branch: refs/heads/master
Commit: 8158f884854a4294d7444a393f28c19d2860
Parents: 552dbaf
Author: Eugene Koifman 
Authored: Fri May 4 13:22:07 2018 -0700
Committer: Eugene Koifman 
Committed: Fri May 4 13:22:07 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  4 ++
 .../hadoop/hive/ql/hooks/WriteEntity.java   | 13 +++--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java|  6 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   | 12 ++--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java  | 58 +---
 .../alter_rename_partition_failure3.q.out   |  2 +-
 .../alter_table_add_partition.q.out |  2 +-
 .../clientnegative/alter_view_failure5.q.out|  2 +-
 .../clientnegative/alter_view_failure7.q.out|  2 +-
 .../test/results/clientnegative/analyze1.q.out  |  2 +-
 .../clientnegative/archive_partspec1.q.out  |  2 +-
 .../test/results/clientnegative/dyn_part1.q.out |  2 +-
 ql/src/test/results/clientnegative/touch2.q.out |  2 +-
 .../truncate_partition_column.q.out |  2 +-
 15 files changed, 84 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 844a200..7628c91 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2436,6 +2436,10 @@ public class HiveConf extends Configuration {
 "In nonstrict mode, for non-ACID resources, INSERT will only acquire 
shared lock, which\n" +
 "allows two concurrent writes to the same partition but still lets 
lock manager prevent\n" +
 "DROP TABLE etc. when the table is being written to"),
+TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true,
+"Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire 
Exclusive locks for\b" +
+"transactional tables.  This ensures that inserts (w/o overwrite) 
running concurrently\n" +
+"are not hidden by the INSERT OVERWRITE."),
 /**
  * @deprecated Use MetastoreConf.TXN_TIMEOUT
  */

http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index c465adf..f1cf113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -63,7 +63,7 @@ public class WriteEntity extends Entity implements 
Serializable {
 
   public WriteEntity(Database database, WriteType type) {
 super(database, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -74,12 +74,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Table t, WriteType type) {
 super(t, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(Table t, WriteType type, boolean complete) {
 super(t, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -104,12 +104,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Partition p, WriteType type) {
 super(p, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(DummyPartition p, WriteType type, boolean complete) {
 super(p, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -161,6 +161,9 @@ public class WriteEntity extends Entity implements 
Serializable {
* @param type new operation type
*/
   public void setWriteType(WriteType type) {
+setWriteTypeInternal(type);
+  }
+  private void setWriteTypeInternal(WriteType type) {
 writeType = type;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/8158f884/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
--
diff --git a/ql/

hive git commit: HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene Koifman, reviewed by Sergey Shelukhin)

2018-05-04 Thread ekoifman 
Repository: hive
Updated Branches:
  refs/heads/branch-3 1ad644c60 -> 812ae7388


HIVE-18570 - ACID IOW implemented using base may delete too much data (Eugene 
Koifman, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/812ae738
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/812ae738
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/812ae738

Branch: refs/heads/branch-3
Commit: 812ae7388936d3a8861304a66eb1f448b126e408
Parents: 1ad644c
Author: Eugene Koifman 
Authored: Fri May 4 13:21:00 2018 -0700
Committer: Eugene Koifman 
Committed: Fri May 4 13:21:00 2018 -0700

--
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  4 ++
 .../hadoop/hive/ql/hooks/WriteEntity.java   | 13 +++--
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java|  6 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   | 12 ++--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java  | 58 +---
 .../alter_rename_partition_failure3.q.out   |  2 +-
 .../alter_table_add_partition.q.out |  2 +-
 .../clientnegative/alter_view_failure5.q.out|  2 +-
 .../clientnegative/alter_view_failure7.q.out|  2 +-
 .../test/results/clientnegative/analyze1.q.out  |  2 +-
 .../clientnegative/archive_partspec1.q.out  |  2 +-
 .../test/results/clientnegative/dyn_part1.q.out |  2 +-
 ql/src/test/results/clientnegative/touch2.q.out |  2 +-
 .../truncate_partition_column.q.out |  2 +-
 15 files changed, 84 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 9a0d8a9..8c6cb61 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2436,6 +2436,10 @@ public class HiveConf extends Configuration {
 "In nonstrict mode, for non-ACID resources, INSERT will only acquire 
shared lock, which\n" +
 "allows two concurrent writes to the same partition but still lets 
lock manager prevent\n" +
 "DROP TABLE etc. when the table is being written to"),
+TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true,
+"Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire 
Exclusive locks for\b" +
+"transactional tables.  This ensures that inserts (w/o overwrite) 
running concurrently\n" +
+"are not hidden by the INSERT OVERWRITE."),
 /**
  * @deprecated Use MetastoreConf.TXN_TIMEOUT
  */

http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java 
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index c465adf..f1cf113 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -63,7 +63,7 @@ public class WriteEntity extends Entity implements 
Serializable {
 
   public WriteEntity(Database database, WriteType type) {
 super(database, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -74,12 +74,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Table t, WriteType type) {
 super(t, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(Table t, WriteType type, boolean complete) {
 super(t, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -104,12 +104,12 @@ public class WriteEntity extends Entity implements 
Serializable {
*/
   public WriteEntity(Partition p, WriteType type) {
 super(p, true);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   public WriteEntity(DummyPartition p, WriteType type, boolean complete) {
 super(p, complete);
-writeType = type;
+setWriteTypeInternal(type);
   }
 
   /**
@@ -161,6 +161,9 @@ public class WriteEntity extends Entity implements 
Serializable {
* @param type new operation type
*/
   public void setWriteType(WriteType type) {
+setWriteTypeInternal(type);
+  }
+  private void setWriteTypeInternal(WriteType type) {
 writeType = type;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/812ae738/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
--
diff --git a