http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index c8bc119..7c66955 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -137,7 +137,6 @@ public class TestTxnCommands { public void tearDown() throws Exception { try { if (d != null) { - runStatementOnDriver("set autocommit true"); dropTables(); d.destroy(); d.close(); @@ -194,7 +193,6 @@ public class TestTxnCommands { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); //List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); //Assert.assertEquals("Data didn't match in autocommit=true (rs)", stringifyValues(rows1), rs); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); @@ -206,8 +204,8 @@ public class TestTxnCommands { dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 2, 0); runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); - runStatementOnDriver("COMMIT");//txn started implicitly by previous statement - runStatementOnDriver("set autocommit true"); + CommandProcessorResponse cpr = runStatementOnDriverNegative("COMMIT");//txn started implicitly by previous statement + Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr.getErrorCode()); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs1); } @@ -218,43 +216,35 @@ public class TestTxnCommands { */ @Test public void testErrors() throws Exception { - runStatementOnDriver("set autocommit true"); - CommandProcessorResponse cpr = runStatementOnDriverNegative("start transaction"); - Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(), cpr.getErrorCode()); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("start transaction"); CommandProcessorResponse cpr2 = runStatementOnDriverNegative("create table foo(x int, y int)"); Assert.assertEquals("Expected DDL to fail in an open txn", ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(), cpr2.getErrorCode()); - runStatementOnDriver("set autocommit true"); CommandProcessorResponse cpr3 = runStatementOnDriverNegative("update " + Table.ACIDTBL + " set a = 1 where b != 1"); Assert.assertEquals("Expected update of bucket column to fail", "FAILED: SemanticException [Error 10302]: Updating values of bucketing columns is not supported. Column a.", cpr3.getErrorMessage()); - //line below should in principle work but Driver doesn't propagate errorCode properly - //Assert.assertEquals("Expected update of bucket column to fail", ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE.getErrorCode(), cpr3.getErrorCode()); - cpr3 = runStatementOnDriverNegative("commit work");//not allowed in AC=true - Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(), cpr.getErrorCode()); - cpr3 = runStatementOnDriverNegative("rollback work");//not allowed in AC=true - Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(), cpr.getErrorCode()); - runStatementOnDriver("set autocommit false"); + Assert.assertEquals("Expected update of bucket column to fail", + ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE.getErrorCode(), cpr3.getErrorCode()); cpr3 = runStatementOnDriverNegative("commit");//not allowed in w/o tx - Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(), cpr.getErrorCode()); + Assert.assertEquals("Error didn't match: " + cpr3, + ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr3.getErrorCode()); cpr3 = runStatementOnDriverNegative("rollback");//not allowed in w/o tx - Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(), cpr.getErrorCode()); + Assert.assertEquals("Error didn't match: " + cpr3, + ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr3.getErrorCode()); runStatementOnDriver("start transaction"); cpr3 = runStatementOnDriverNegative("start transaction");//not allowed in a tx - Assert.assertEquals("Expected start transaction to fail", ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(), cpr3.getErrorCode()); + Assert.assertEquals("Expected start transaction to fail", + ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(), cpr3.getErrorCode()); runStatementOnDriver("start transaction");//ok since previously opened txn was killed runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Can't see my own write", 1, rs0.size()); - runStatementOnDriver("set autocommit true");//this should commit previous txn + runStatementOnDriver("commit work"); rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Can't see my own write", 1, rs0.size()); } @Test public void testReadMyOwnInsert() throws Exception { - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL); Assert.assertEquals("Expected empty " + Table.ACIDTBL, 0, rs.size()); @@ -269,7 +259,6 @@ public class TestTxnCommands { } @Test public void testImplicitRollback() throws Exception { - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); @@ -286,18 +275,15 @@ public class TestTxnCommands { } @Test public void testExplicitRollback() throws Exception { - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)"); runStatementOnDriver("ROLLBACK"); - runStatementOnDriver("set autocommit true"); List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Rollback didn't rollback", 0, rs.size()); } @Test public void testMultipleInserts() throws Exception { - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); int[][] rows1 = {{1,2},{3,4}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); @@ -310,7 +296,6 @@ public class TestTxnCommands { runStatementOnDriver("commit"); dumpTableData(Table.ACIDTBL, 1, 0); dumpTableData(Table.ACIDTBL, 1, 1); - runStatementOnDriver("set autocommit true"); List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match after commit rs1", allData, rs1); } @@ -320,14 +305,12 @@ public class TestTxnCommands { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4"); int[][] updatedData2 = {{1,2}}; List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3); runStatementOnDriver("commit"); - runStatementOnDriver("set autocommit true"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4); } @@ -338,7 +321,6 @@ public class TestTxnCommands { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); @@ -351,7 +333,6 @@ public class TestTxnCommands { List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after update", stringifyValues(updatedData), rs2); runStatementOnDriver("commit"); - runStatementOnDriver("set autocommit true"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData), rs4); } @@ -361,7 +342,6 @@ public class TestTxnCommands { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); int[][] rows2 = {{5,6},{7,8}}; runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2)); @@ -382,7 +362,6 @@ public class TestTxnCommands { List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3); runStatementOnDriver("commit"); - runStatementOnDriver("set autocommit true"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4); } @@ -392,7 +371,6 @@ public class TestTxnCommands { runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1)); List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0); - runStatementOnDriver("set autocommit false"); runStatementOnDriver("START TRANSACTION"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 8"); int[][] updatedData2 = {{1,2},{3,4},{5,6}}; @@ -412,7 +390,6 @@ public class TestTxnCommands { int [][] updatedData4 = {{1,3},{5,3}}; Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData4), rs5); runStatementOnDriver("commit"); - runStatementOnDriver("set autocommit true"); List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b"); Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData4), rs4); } @@ -433,7 +410,6 @@ public class TestTxnCommands { } @Test public void testTimeOutReaper() throws Exception { - runStatementOnDriver("set autocommit false"); runStatementOnDriver("start transaction"); runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5"); //make sure currently running txn is considered aborted by housekeeper @@ -467,7 +443,7 @@ public class TestTxnCommands { } } Assert.assertNotNull(txnInfo); - Assert.assertEquals(2, txnInfo.getId()); + Assert.assertEquals(12, txnInfo.getId()); Assert.assertEquals(TxnState.OPEN, txnInfo.getState()); String s =TxnDbUtil.queryToString("select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false); String[] vals = s.split("\\s+");
http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index e2db5b7..5786c4f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -662,11 +662,11 @@ public class TestTxnCommands2 { FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER); Arrays.sort(buckets); if (numDelta == 1) { - Assert.assertEquals("delta_0000001_0000001_0000", status[i].getPath().getName()); + Assert.assertEquals("delta_0000022_0000022_0000", status[i].getPath().getName()); Assert.assertEquals(BUCKET_COUNT - 1, buckets.length); Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } else if (numDelta == 2) { - Assert.assertEquals("delta_0000002_0000002_0000", status[i].getPath().getName()); + Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName()); Assert.assertEquals(BUCKET_COUNT, buckets.length); Assert.assertEquals("bucket_00000", buckets[0].getPath().getName()); Assert.assertEquals("bucket_00001", buckets[1].getPath().getName()); @@ -711,7 +711,7 @@ public class TestTxnCommands2 { Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } else if (numBase == 2) { // The new base dir now has two bucket files, since the delta dir has two bucket files - Assert.assertEquals("base_0000002", status[i].getPath().getName()); + Assert.assertEquals("base_0000023", status[i].getPath().getName()); Assert.assertEquals(BUCKET_COUNT, buckets.length); Assert.assertEquals("bucket_00000", buckets[0].getPath().getName()); Assert.assertEquals("bucket_00001", buckets[1].getPath().getName()); @@ -738,7 +738,7 @@ public class TestTxnCommands2 { status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); Assert.assertEquals(1, status.length); - Assert.assertEquals("base_0000002", status[0].getPath().getName()); + Assert.assertEquals("base_0000023", status[0].getPath().getName()); FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER); Arrays.sort(buckets); Assert.assertEquals(BUCKET_COUNT, buckets.length); @@ -760,11 +760,6 @@ public class TestTxnCommands2 { runStatementOnDriver("select * from " + Table.NONACIDORCTBL); String value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY); Assert.assertNull("The entry should be null for query that doesn't involve ACID tables", value); - - // 2. Run a query against an ACID table, and we should have txn logged in conf - runStatementOnDriver("select * from " + Table.ACIDTBL); - value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY); - Assert.assertNotNull("The entry shouldn't be null for query that involves ACID tables", value); } @Test @@ -773,10 +768,14 @@ public class TestTxnCommands2 { int[][] tableData = {{1,2},{3,3}}; runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData)); int[][] tableData2 = {{5,3}}; + //this will cause next txn to be marked aborted but the data is still written to disk + hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true); runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData2)); - hiveConf.set(ValidTxnList.VALID_TXNS_KEY, "0:"); + assert hiveConf.get(ValidTxnList.VALID_TXNS_KEY) == null : "previous txn should've cleaned it"; + //so now if HIVEFETCHTASKCONVERSION were to use a stale value, it would use a + //ValidTxnList with HWM=MAX_LONG, i.e. include the data for aborted txn List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL); - Assert.assertEquals("Missing data", 3, rs.size()); + Assert.assertEquals("Extra data", 2, rs.size()); } @Test public void testUpdateMixedCase() throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java index e516f18..ea5ecbc 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java @@ -84,7 +84,6 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { runStatementOnDriver("create table acidTblLegacy (a int, b int) clustered by (a) into " + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')"); runStatementOnDriver("alter table acidTblLegacy SET TBLPROPERTIES ('transactional_properties' = 'default')"); } - /** * Test the query correctness and directory layout for ACID table conversion with split-update * enabled. @@ -96,7 +95,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { * @throws Exception */ @Test - public void testNonAcidToAcidSplitUpdateConversion1() throws Exception { + @Override + public void testNonAcidToAcidConversion1() throws Exception { FileSystem fs = FileSystem.get(hiveConf); FileStatus[] status; @@ -226,7 +226,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { * @throws Exception */ @Test - public void testNonAcidToAcidSplitUpdateConversion2() throws Exception { + @Override + public void testNonAcidToAcidConversion2() throws Exception { FileSystem fs = FileSystem.get(hiveConf); FileStatus[] status; @@ -360,7 +361,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { * @throws Exception */ @Test - public void testNonAcidToAcidSplitUpdateConversion3() throws Exception { + @Override + public void testNonAcidToAcidConversion3() throws Exception { FileSystem fs = FileSystem.get(hiveConf); FileStatus[] status; @@ -442,11 +444,11 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER); Arrays.sort(buckets); if (numDelta == 1) { - Assert.assertEquals("delta_0000001_0000001_0000", status[i].getPath().getName()); + Assert.assertEquals("delta_0000022_0000022_0000", status[i].getPath().getName()); Assert.assertEquals(BUCKET_COUNT - 1, buckets.length); Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } else if (numDelta == 2) { - Assert.assertEquals("delta_0000002_0000002_0000", status[i].getPath().getName()); + Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName()); Assert.assertEquals(1, buckets.length); Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } @@ -455,7 +457,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER); Arrays.sort(buckets); if (numDeleteDelta == 1) { - Assert.assertEquals("delete_delta_0000001_0000001_0000", status[i].getPath().getName()); + Assert.assertEquals("delete_delta_0000022_0000022_0000", status[i].getPath().getName()); Assert.assertEquals(BUCKET_COUNT - 1, buckets.length); Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } @@ -502,7 +504,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } else if (numBase == 2) { // The new base dir now has two bucket files, since the delta dir has two bucket files - Assert.assertEquals("base_0000002", status[i].getPath().getName()); + Assert.assertEquals("base_0000023", status[i].getPath().getName()); Assert.assertEquals(1, buckets.length); Assert.assertEquals("bucket_00001", buckets[0].getPath().getName()); } @@ -528,7 +530,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 { status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER); Assert.assertEquals(1, status.length); - Assert.assertEquals("base_0000002", status[0].getPath().getName()); + Assert.assertEquals("base_0000023", status[0].getPath().getName()); FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER); Arrays.sort(buckets); Assert.assertEquals(1, buckets.length); http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java index 9bfc7d1..14ff58e 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java @@ -36,11 +36,13 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.HiveOperation; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.util.ArrayList; @@ -79,13 +81,14 @@ public class TestDbTxnManager { @Test public void testSingleReadTable() throws Exception { addTableInput(); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -93,13 +96,14 @@ public class TestDbTxnManager { @Test public void testSingleReadPartition() throws Exception { addPartitionInput(newTable(true)); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); + txnMgr.openTxn(ctx, null); txnMgr.acquireLocks(qp, ctx, null); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); @@ -111,13 +115,14 @@ public class TestDbTxnManager { addPartitionInput(t); addPartitionInput(t); addPartitionInput(t); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(3, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -129,13 +134,14 @@ public class TestDbTxnManager { addPartitionInput(t); addPartitionInput(t); addTableInput(); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(4, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -143,7 +149,7 @@ public class TestDbTxnManager { @Test public void testSingleWriteTable() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -159,7 +165,7 @@ public class TestDbTxnManager { @Test public void testSingleWritePartition() throws Exception { WriteEntity we = addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -174,7 +180,7 @@ public class TestDbTxnManager { @Test public void testWriteDynamicPartition() throws Exception { WriteEntity we = addDynamicPartitionedOutput(newTable(true), WriteEntity.WriteType.INSERT); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -217,7 +223,7 @@ public class TestDbTxnManager { @Test public void testExceptions() throws Exception { addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); ((DbTxnManager) txnMgr).openTxn(ctx, "NicholasII", HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS) * 2); Thread.sleep(HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS)); runReaper(); @@ -243,10 +249,11 @@ public class TestDbTxnManager { txnMgr.rollbackTxn();//this is idempotent } + @Ignore("This seems useless now that we have a txn for everything") @Test public void testLockTimeout() throws Exception { addPartitionInput(newTable(true)); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); //make sure it works with nothing to expire testLockExpiration(txnMgr, 0, true); @@ -294,7 +301,7 @@ public class TestDbTxnManager { addPartitionInput(t); addPartitionInput(t); WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -309,7 +316,7 @@ public class TestDbTxnManager { @Test public void testUpdate() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.UPDATE); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -324,7 +331,7 @@ public class TestDbTxnManager { @Test public void testDelete() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -339,7 +346,7 @@ public class TestDbTxnManager { @Test public void testRollback() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); @@ -354,13 +361,14 @@ public class TestDbTxnManager { @Test public void testDDLExclusive() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_EXCLUSIVE); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.DROPTABLE); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.rollbackTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -368,13 +376,14 @@ public class TestDbTxnManager { @Test public void testDDLShared() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_SHARED); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.ALTERTABLE_EXCHANGEPARTITION); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); Assert.assertEquals(1, TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId)); - txnMgr.getLockManager().unlock(locks.get(0)); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -382,10 +391,12 @@ public class TestDbTxnManager { @Test public void testDDLNoLock() throws Exception { WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_NO_LOCK); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.CREATEDATABASE); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertNull(locks); + txnMgr.rollbackTxn(); } @Test @@ -406,11 +417,12 @@ public class TestDbTxnManager { @Test public void testLockAcquisitionAndRelease() throws Exception { addTableInput(); - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); + txnMgr.openTxn(ctx, "fred"); txnMgr.acquireLocks(qp, ctx, "fred"); List<HiveLock> locks = ctx.getHiveLocks(); Assert.assertEquals(1, locks.size()); - txnMgr.releaseLocks(locks); + txnMgr.commitTxn(); locks = txnMgr.getLockManager().getLocks(false, false); Assert.assertEquals(0, locks.size()); } @@ -421,7 +433,7 @@ public class TestDbTxnManager { addTableInput(); LockException exception = null; - QueryPlan qp = new MockQueryPlan(this); + QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY); // Case 1: If there's no delay for the heartbeat, txn should be able to commit txnMgr.openTxn(ctx, "fred"); @@ -493,7 +505,8 @@ public class TestDbTxnManager { private final HashSet<WriteEntity> outputs = new HashSet<>(); private final String queryId; - MockQueryPlan(TestDbTxnManager test) { + MockQueryPlan(TestDbTxnManager test, HiveOperation operation) { + super(operation); inputs.addAll(test.readEntities); outputs.addAll(test.writeEntities); queryId = makeQueryId();
