Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java?rev=1654899&r1=1654898&r2=1654899&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java Mon Jan 26 21:21:45 2015 @@ -17,19 +17,20 @@ */ package org.apache.hadoop.hive.ql.txn.compactor; -import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.*; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -274,7 +275,6 @@ public class TestWorker extends Compacto // There should still now be 5 directories in the location FileSystem fs = FileSystem.get(conf); FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); - for (int i = 0; i < stat.length; i++) System.out.println("HERE: " + stat[i].getPath().toString()); Assert.assertEquals(4, stat.length); // Find the new delta file and make sure it has the right contents @@ -296,6 +296,78 @@ public class TestWorker extends Compacto } @Test + public void minorWithOpenInMiddle() throws Exception { + LOG.debug("Starting minorWithOpenInMiddle"); + Table t = newTable("default", "mtwb", false); + + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 25L, 3); + addLengthFile(t, null, 23L, 25L, 3); + addDeltaFile(t, null, 26L, 27L, 2); + burnThroughTransactions(27, new HashSet<Long>(Arrays.asList(23L)), null); + + CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MINOR); + txnHandler.compact(rqst); + + startWorker(); + + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List<ShowCompactResponseElement> compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + Assert.assertEquals("ready for cleaning", compacts.get(0).getState()); + + // There should still now be 5 directories in the location + FileSystem fs = FileSystem.get(conf); + FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + Assert.assertEquals(5, stat.length); + + // Find the new delta file and make sure it has the right contents + Arrays.sort(stat); + Assert.assertEquals("base_20", stat[0].getPath().getName()); + Assert.assertEquals("delta_0000021_0000022", stat[1].getPath().getName()); + Assert.assertEquals("delta_21_22", stat[2].getPath().getName()); + Assert.assertEquals("delta_23_25", stat[3].getPath().getName()); + Assert.assertEquals("delta_26_27", stat[4].getPath().getName()); + } + + @Test + public void minorWithAborted() throws Exception { + LOG.debug("Starting minorWithAborted"); + Table t = newTable("default", "mtwb", false); + + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 25L, 3); + addLengthFile(t, null, 23L, 25L, 3); + addDeltaFile(t, null, 26L, 27L, 2); + burnThroughTransactions(27, null, new HashSet<Long>(Arrays.asList(24L, 25L))); + + CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MINOR); + txnHandler.compact(rqst); + + startWorker(); + + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List<ShowCompactResponseElement> compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + Assert.assertEquals("ready for cleaning", compacts.get(0).getState()); + + // There should still now be 5 directories in the location + FileSystem fs = FileSystem.get(conf); + FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + Assert.assertEquals(5, stat.length); + + // Find the new delta file and make sure it has the right contents + Arrays.sort(stat); + Assert.assertEquals("base_20", stat[0].getPath().getName()); + Assert.assertEquals("delta_0000021_0000027", stat[1].getPath().getName()); + Assert.assertEquals("delta_21_22", stat[2].getPath().getName()); + Assert.assertEquals("delta_23_25", stat[3].getPath().getName()); + Assert.assertEquals("delta_26_27", stat[4].getPath().getName()); + } + + @Test public void minorPartitionWithBase() throws Exception { Table t = newTable("default", "mpwb", true); Partition p = newPartition(t, "today"); @@ -481,7 +553,7 @@ public class TestWorker extends Compacto addDeltaFile(t, null, 1L, 2L, 2); addDeltaFile(t, null, 3L, 4L, 2); - burnThroughTransactions(5); + burnThroughTransactions(4); CompactionRequest rqst = new CompactionRequest("default", "matnb", CompactionType.MAJOR); txnHandler.compact(rqst); @@ -603,6 +675,7 @@ public class TestWorker extends Compacto @Test public void majorPartitionWithBaseMissingBuckets() throws Exception { + LOG.debug("Starting majorPartitionWithBaseMissingBuckets"); Table t = newTable("default", "mapwbmb", true); Partition p = newPartition(t, "today"); @@ -611,7 +684,7 @@ public class TestWorker extends Compacto addDeltaFile(t, p, 21L, 22L, 2, 2, false); addDeltaFile(t, p, 23L, 26L, 4); - burnThroughTransactions(25); + burnThroughTransactions(27); CompactionRequest rqst = new CompactionRequest("default", "mapwbmb", CompactionType.MAJOR); rqst.setPartitionname("ds=today"); @@ -654,4 +727,76 @@ public class TestWorker extends Compacto } Assert.assertTrue(sawNewBase); } + + @Test + public void majorWithOpenInMiddle() throws Exception { + LOG.debug("Starting majorWithOpenInMiddle"); + Table t = newTable("default", "mtwb", false); + + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 25L, 3); + addLengthFile(t, null, 23L, 25L, 3); + addDeltaFile(t, null, 26L, 27L, 2); + burnThroughTransactions(27, new HashSet<Long>(Arrays.asList(23L)), null); + + CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MAJOR); + txnHandler.compact(rqst); + + startWorker(); + + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List<ShowCompactResponseElement> compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + Assert.assertEquals("ready for cleaning", compacts.get(0).getState()); + + // There should still now be 5 directories in the location + FileSystem fs = FileSystem.get(conf); + FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + Assert.assertEquals(5, stat.length); + + // Find the new delta file and make sure it has the right contents + Arrays.sort(stat); + Assert.assertEquals("base_0000022", stat[0].getPath().getName()); + Assert.assertEquals("base_20", stat[1].getPath().getName()); + Assert.assertEquals("delta_21_22", stat[2].getPath().getName()); + Assert.assertEquals("delta_23_25", stat[3].getPath().getName()); + Assert.assertEquals("delta_26_27", stat[4].getPath().getName()); + } + + @Test + public void majorWithAborted() throws Exception { + LOG.debug("Starting majorWithAborted"); + Table t = newTable("default", "mtwb", false); + + addBaseFile(t, null, 20L, 20); + addDeltaFile(t, null, 21L, 22L, 2); + addDeltaFile(t, null, 23L, 25L, 3); + addLengthFile(t, null, 23L, 25L, 3); + addDeltaFile(t, null, 26L, 27L, 2); + burnThroughTransactions(27, null, new HashSet<Long>(Arrays.asList(24L, 25L))); + + CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MAJOR); + txnHandler.compact(rqst); + + startWorker(); + + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List<ShowCompactResponseElement> compacts = rsp.getCompacts(); + Assert.assertEquals(1, compacts.size()); + Assert.assertEquals("ready for cleaning", compacts.get(0).getState()); + + // There should still now be 5 directories in the location + FileSystem fs = FileSystem.get(conf); + FileStatus[] stat = fs.listStatus(new Path(t.getSd().getLocation())); + Assert.assertEquals(5, stat.length); + + // Find the new delta file and make sure it has the right contents + Arrays.sort(stat); + Assert.assertEquals("base_0000027", stat[0].getPath().getName()); + Assert.assertEquals("base_20", stat[1].getPath().getName()); + Assert.assertEquals("delta_21_22", stat[2].getPath().getName()); + Assert.assertEquals("delta_23_25", stat[3].getPath().getName()); + Assert.assertEquals("delta_26_27", stat[4].getPath().getName()); + } }