This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2d444fa HIVE-20801: ACID: Allow DbTxnManager to ignore non-ACID table
read locking (Gopal Vijayaraghavan, reviewed by Eugene Koifman, Ashutosh
Chauhan, Denys Kuzmenko)
2d444fa is described below
commit 2d444fafa2c2cc0d0f5aa45378536398c64e1f7b
Author: Peter Vary <pv...@cloudera.com>
AuthorDate: Fri Jan 24 10:05:05 2020 +0100
HIVE-20801: ACID: Allow DbTxnManager to ignore non-ACID table read locking
(Gopal Vijayaraghavan, reviewed by Eugene Koifman, Ashutosh Chauhan, Denys
Kuzmenko)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 11 +++--
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 51 ++++++++++++---------
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 52 ++++++++++++++++++++++
3 files changed, 89 insertions(+), 25 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index b79515f..d8aabd8 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2644,12 +2644,17 @@ public class HiveConf extends Configuration {
"In nonstrict mode, for non-ACID resources, INSERT will only acquire
shared lock, which\n" +
"allows two concurrent writes to the same partition but still lets
lock manager prevent\n" +
"DROP TABLE etc. when the table is being written to"),
+ HIVE_TXN_NONACID_READ_LOCKS("hive.txn.nonacid.read.locks", true,
+ "Flag to turn off the read locks for non-ACID tables, when set to
false.\n" +
+ "Could be exercised to improve the performance of non-ACID tables in
clusters where read locking " +
+ "is enabled globally to support ACID. Can cause issues with concurrent
DDL operations, or slow S3 writes."),
HIVE_TXN_READ_LOCKS("hive.txn.read.locks", true,
- "flag to turn off the strict read lock when set to false"),
+ "Flag to turn off the read locks, when set to false. Although its not
recommended, \n" +
+ "but in performance critical scenarios this option may be exercised."),
TXN_OVERWRITE_X_LOCK("hive.txn.xlock.iow", true,
"Ensures commands with OVERWRITE (such as INSERT OVERWRITE) acquire
Exclusive locks for\n" +
- "transactional tables. This ensures that inserts (w/o overwrite)
running concurrently\n" +
- "are not hidden by the INSERT OVERWRITE."),
+ "transactional tables. This ensures that inserts (w/o overwrite)
running concurrently\n" +
+ "are not hidden by the INSERT OVERWRITE."),
HIVE_TXN_STATS_ENABLED("hive.txn.stats.enabled", true,
"Whether Hive supports transactional stats (accurate stats for
transactional tables)"),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index ed68e0f..2f5ec52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -2845,8 +2845,10 @@ public class AcidUtils {
public static List<LockComponent> makeLockComponents(Set<WriteEntity>
outputs, Set<ReadEntity> inputs,
HiveConf conf) {
List<LockComponent> lockComponents = new ArrayList<>();
- // For each source to read, get a shared lock
boolean skipReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_READ_LOCKS);
+ boolean skipNonAcidReadLock =
!conf.getBoolVar(ConfVars.HIVE_TXN_NONACID_READ_LOCKS);
+
+ // For each source to read, get a shared lock
for (ReadEntity input : inputs) {
if (input.isDummy()
|| !input.needsLock()
@@ -2863,33 +2865,38 @@ public class AcidUtils {
Table t = null;
switch (input.getType()) {
- case DATABASE:
- compBuilder.setDbName(input.getDatabase().getName());
- break;
-
- case TABLE:
- t = input.getTable();
- compBuilder.setDbName(t.getDbName());
- compBuilder.setTableName(t.getTableName());
- break;
-
- case PARTITION:
- case DUMMYPARTITION:
- compBuilder.setPartitionName(input.getPartition().getName());
- t = input.getPartition().getTable();
- compBuilder.setDbName(t.getDbName());
- compBuilder.setTableName(t.getTableName());
- break;
-
- default:
- // This is a file or something we don't hold locks for.
+ case DATABASE:
+ compBuilder.setDbName(input.getDatabase().getName());
+ break;
+
+ case TABLE:
+ t = input.getTable();
+ compBuilder.setDbName(t.getDbName());
+ compBuilder.setTableName(t.getTableName());
+ break;
+
+ case PARTITION:
+ case DUMMYPARTITION:
+ compBuilder.setPartitionName(input.getPartition().getName());
+ t = input.getPartition().getTable();
+ compBuilder.setDbName(t.getDbName());
+ compBuilder.setTableName(t.getTableName());
+ break;
+
+ default:
+ // This is a file or something we don't hold locks for.
+ continue;
+ }
+ if (skipNonAcidReadLock && !AcidUtils.isTransactionalTable(t)) {
+ // skip read-locks for non-transactional tables
+ // read-locks don't protect non-transactional tables data consistency
continue;
}
if (t != null) {
compBuilder.setIsTransactional(AcidUtils.isTransactionalTable(t));
}
LockComponent comp = compBuilder.build();
- LOG.debug("Adding lock component to lock request " + comp.toString());
+ LOG.debug("Adding lock component to lock request {} ", comp);
lockComponents.add(comp);
}
// For each source to write to, get the appropriate lock type. If it's
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 50859d1..48e9afc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -711,6 +711,58 @@ public class TestDbTxnManager2 {
conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
}
+ /**
+ * Check to make sure we acquire proper locks for queries involving
non-strict locking
+ */
+ @Test
+ public void checkExpectedReadLocksForNonAcidTables() throws Exception {
+ dropTable(new String[] {"tab_acid", "tab_not_acid"});
+ driver.run("create table if not exists tab_acid (a int, b int) partitioned
by (p string) " +
+ "clustered by (a) into 2 buckets stored as orc TBLPROPERTIES
('transactional'='true')");
+ driver.run("create table if not exists tab_not_acid (na int, nb int)
partitioned by (np string) " +
+ "clustered by (na) into 2 buckets stored as orc TBLPROPERTIES
('transactional'='false')");
+ driver.run("insert into tab_acid partition(p) (a,b,p)
values(1,2,'foo'),(3,4,'bar')");
+ driver.run("insert into tab_not_acid partition(np) (na,nb,np)
values(1,2,'blah'),(3,4,'doh')");
+
+ // Test non-acid read-locking mode - the read locks are only obtained for
the ACID side
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS, false);
+
+ HiveTxnManager txnMgr1 =
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr1.openTxn(ctx, "T1");
+ driver.compileAndRespond("select * from tab_acid inner join tab_not_acid
on a = na", true);
+ txnMgr1.acquireLocks(driver.getPlan(), ctx, "T1");
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr1);
+ Assert.assertEquals("Unexpected lock count", 3, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=bar", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=foo", locks);
+
+ HiveTxnManager txnMgr2 =
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr2.openTxn(ctx, "T2");
+ driver.compileAndRespond("insert into tab_not_acid partition(np='doh')
values(5,6)", true);
+ LockState ls = ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx,
"T2", false);
+ locks = getLocks(txnMgr2);
+ Assert.assertEquals("Unexpected lock count", 4, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=bar", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=foo", locks);
+ checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default",
"tab_not_acid", "np=doh", locks);
+
+ HiveTxnManager txnMgr3 =
TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr3.openTxn(ctx, "T3");
+ driver.compileAndRespond("insert into tab_not_acid partition(np='blah')
values(7,8)", true);
+ ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false);
+ locks = getLocks(txnMgr3);
+ Assert.assertEquals("Unexpected lock count", 5, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=bar", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid",
"p=foo", locks);
+ checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default",
"tab_not_acid", "np=blah", locks);
+
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS,
+ HiveConf.ConfVars.HIVE_TXN_NONACID_READ_LOCKS.defaultBoolVal);
+ }
+
@Test
public void testLockingOnInsertIntoNonNativeTables() throws Exception {
dropTable(new String[] {"tab_not_acid"});
