Repository: hbase Updated Branches: refs/heads/master 5aab961a7 -> 5e26ae043
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java index d891c20..a2c4d3c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java @@ -148,143 +148,12 @@ public class MobSnapshotTestingUtils { // ========================================================================== // Snapshot Mock // ========================================================================== - public static class SnapshotMock { - private final static String TEST_FAMILY = "cf"; - public final static int TEST_NUM_REGIONS = 4; - - private final Configuration conf; - private final FileSystem fs; - private final Path rootDir; - - static class RegionData { - public HRegionInfo hri; - public Path tableDir; - public Path[] files; - - public RegionData(final Path tableDir, final HRegionInfo hri, - final int nfiles) { - this.tableDir = tableDir; - this.hri = hri; - this.files = new Path[nfiles]; - } - } - - public static class SnapshotBuilder { - private final RegionData[] tableRegions; - private final SnapshotDescription desc; - private final HTableDescriptor htd; - private final Configuration conf; - private final FileSystem fs; - private final Path rootDir; - private Path snapshotDir; - private int snapshotted = 0; - - public SnapshotBuilder(final Configuration conf, final FileSystem fs, - final Path rootDir, final HTableDescriptor htd, - final SnapshotDescription desc, final RegionData[] tableRegions) - throws IOException { - this.fs = fs; - this.conf = conf; - this.rootDir = rootDir; - this.htd = htd; - this.desc = desc; - this.tableRegions = tableRegions; - this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, - rootDir); - new FSTableDescriptors(conf).createTableDescriptorForTableDirectory( - snapshotDir, new TableDescriptor(htd), false); - } - - public HTableDescriptor getTableDescriptor() { - return this.htd; - } - - public SnapshotDescription getSnapshotDescription() { - return this.desc; - } - - public Path getSnapshotsDir() { - return this.snapshotDir; - } - - public Path[] addRegion() throws IOException { - return addRegion(desc); - } - - public Path[] addRegionV1() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION).build()); - } - - public Path[] addRegionV2() throws IOException { - return addRegion(desc.toBuilder() - .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION).build()); - } - - private Path[] addRegion(final SnapshotDescription desc) - throws IOException { - if (this.snapshotted == tableRegions.length) { - throw new UnsupportedOperationException( - "No more regions in the table"); - } - - RegionData regionData = tableRegions[this.snapshotted++]; - ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher( - desc.getName()); - SnapshotManifest manifest = SnapshotManifest.create(conf, fs, - snapshotDir, desc, monitor); - manifest.addRegion(regionData.tableDir, regionData.hri); - return regionData.files; - } - - public Path commit() throws IOException { - ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher( - desc.getName()); - SnapshotManifest manifest = SnapshotManifest.create(conf, fs, - snapshotDir, desc, monitor); - manifest.addTableDescriptor(htd); - manifest.consolidate(); - SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, - fs); - snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, - rootDir); - return snapshotDir; - } - } - - public SnapshotMock(final Configuration conf, final FileSystem fs, - final Path rootDir) { - this.fs = fs; - this.conf = conf; - this.rootDir = rootDir; - } - - public SnapshotBuilder createSnapshotV1(final String snapshotName) - throws IOException { - return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION); - } - - public SnapshotBuilder createSnapshotV2(final String snapshotName) - throws IOException { - return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION); - } - - private SnapshotBuilder createSnapshot(final String snapshotName, - final int version) throws IOException { - HTableDescriptor htd = createHtd(snapshotName); - - RegionData[] regions = createTable(htd, TEST_NUM_REGIONS); - - SnapshotDescription desc = SnapshotDescription.newBuilder() - .setTable(htd.getNameAsString()).setName(snapshotName) - .setVersion(version).build(); - - Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, - rootDir); - SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); - return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); + public static class SnapshotMock extends SnapshotTestingUtils.SnapshotMock { + public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) { + super(conf, fs, rootDir); } + @Override public HTableDescriptor createHtd(final String tableName) { HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); @@ -293,55 +162,5 @@ public class MobSnapshotTestingUtils { htd.addFamily(hcd); return htd; } - - private RegionData[] createTable(final HTableDescriptor htd, - final int nregions) throws IOException { - Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName()); - new FSTableDescriptors(conf).createTableDescriptorForTableDirectory( - tableDir, new TableDescriptor(htd), false); - - assertTrue(nregions % 2 == 0); - RegionData[] regions = new RegionData[nregions]; - for (int i = 0; i < regions.length; i += 2) { - byte[] startKey = Bytes.toBytes(0 + i * 2); - byte[] endKey = Bytes.toBytes(1 + i * 2); - - // First region, simple with one plain hfile. - HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); - HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem( - conf, fs, tableDir, hri); - regions[i] = new RegionData(tableDir, hri, 3); - for (int j = 0; j < regions[i].files.length; ++j) { - Path storeFile = createStoreFile(rfs.createTempName()); - regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); - } - - // Second region, used to test the split case. - // This region contains a reference to the hfile in the first region. - startKey = Bytes.toBytes(2 + i * 2); - endKey = Bytes.toBytes(3 + i * 2); - hri = new HRegionInfo(htd.getTableName()); - rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, - hri); - regions[i + 1] = new RegionData(tableDir, hri, regions[i].files.length); - for (int j = 0; j < regions[i].files.length; ++j) { - String refName = regions[i].files[j].getName() + '.' - + regions[i].hri.getEncodedName(); - Path refFile = createStoreFile(new Path(rootDir, refName)); - regions[i + 1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile); - } - } - return regions; - } - - private Path createStoreFile(final Path storeFile) throws IOException { - FSDataOutputStream out = fs.create(storeFile); - try { - out.write(Bytes.toBytes(storeFile.toString())); - } finally { - out.close(); - } - return storeFile; - } } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 9caf905..5e8e7d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -448,7 +448,7 @@ public class SnapshotTestingUtils { // Snapshot Mock // ========================================================================== public static class SnapshotMock { - private final static String TEST_FAMILY = "cf"; + protected final static String TEST_FAMILY = "cf"; public final static int TEST_NUM_REGIONS = 4; private final Configuration conf; http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index 9b8bd56..69f1fdb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -68,12 +68,12 @@ public class TestExportSnapshot { protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private final static byte[] FAMILY = Bytes.toBytes("cf"); + protected final static byte[] FAMILY = Bytes.toBytes("cf"); + protected TableName tableName; private byte[] emptySnapshotName; private byte[] snapshotName; private int tableNumFiles; - private TableName tableName; private Admin admin; public static void setUpBaseConf(Configuration conf) { @@ -111,7 +111,7 @@ public class TestExportSnapshot { emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid); // create Table - SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY); + createTable(); // Take an empty snapshot admin.snapshot(emptySnapshotName, tableName); @@ -124,6 +124,10 @@ public class TestExportSnapshot { admin.snapshot(snapshotName, tableName); } + protected void createTable() throws Exception { + SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY); + } + @After public void tearDown() throws Exception { TEST_UTIL.deleteTable(tableName); @@ -356,6 +360,10 @@ public class TestExportSnapshot { assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); } + protected boolean bypassRegion(HRegionInfo regionInfo) { + return false; + } + /* * Verify if the files exists */ @@ -370,6 +378,9 @@ public class TestExportSnapshot { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { + if (bypassRegion(regionInfo)) + return; + String hfile = storeFile.getName(); snapshotFiles.add(hfile); if (storeFile.hasReference()) { http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index b45d676..fca9f16 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -67,28 +67,20 @@ import org.junit.experimental.categories.Category; @Category({RegionServerTests.class, LargeTests.class}) public class TestFlushSnapshotFromClient { private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final int NUM_RS = 2; - private static final byte[] TEST_FAM = Bytes.toBytes("fam"); - private static final TableName TABLE_NAME = TableName.valueOf("test"); - private final int DEFAULT_NUM_ROWS = 100; - /** - * Setup the config for the cluster - * @throws Exception on failure - */ + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + protected static final int NUM_RS = 2; + protected static final byte[] TEST_FAM = Bytes.toBytes("fam"); + protected static final TableName TABLE_NAME = TableName.valueOf("test"); + protected final int DEFAULT_NUM_ROWS = 100; + @BeforeClass public static void setupCluster() throws Exception { - // Uncomment the following lines if more verbosity is needed for - // debugging (see HBASE-12285 for details). - //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)AbstractRpcClient.LOG).getLogger().setLevel(Level.ALL); - //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); setupConf(UTIL.getConfiguration()); UTIL.startMiniCluster(NUM_RS); } - private static void setupConf(Configuration conf) { + protected static void setupConf(Configuration conf) { // disable the ui conf.setInt("hbase.regionsever.info.port", -1); // change the flush size to a small amount, regulating number of store files @@ -107,6 +99,10 @@ public class TestFlushSnapshotFromClient { @Before public void setup() throws Exception { + createTable(); + } + + protected void createTable() throws Exception { SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM); } @@ -344,9 +340,9 @@ public class TestFlushSnapshotFromClient { admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName); - SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows); - SnapshotTestingUtils.verifyRowCount(UTIL, cloneBeforeMergeName, numRows); - SnapshotTestingUtils.verifyRowCount(UTIL, cloneAfterMergeName, numRows); + verifyRowCount(UTIL, TABLE_NAME, numRows); + verifyRowCount(UTIL, cloneBeforeMergeName, numRows); + verifyRowCount(UTIL, cloneAfterMergeName, numRows); // test that we can delete the snapshot UTIL.deleteTable(cloneAfterMergeName); @@ -390,8 +386,8 @@ public class TestFlushSnapshotFromClient { admin.cloneSnapshot(snapshotName, cloneName); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName); - SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows); - SnapshotTestingUtils.verifyRowCount(UTIL, cloneName, numRows); + verifyRowCount(UTIL, TABLE_NAME, numRows); + verifyRowCount(UTIL, cloneName, numRows); // test that we can delete the snapshot UTIL.deleteTable(cloneName); @@ -539,4 +535,14 @@ public class TestFlushSnapshotFromClient { } SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME); } + + + protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, + long expectedRows) throws IOException { + SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows); + } + + protected int countRows(final Table table, final byte[]... families) throws IOException { + return UTIL.countRows(table, families); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java index b7628f17..5535d5e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java @@ -18,69 +18,30 @@ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo; -import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import org.junit.experimental.categories.Category; /** * Test Export Snapshot Tool */ @Category(MediumTests.class) -public class TestMobExportSnapshot { +public class TestMobExportSnapshot extends TestExportSnapshot { private final Log LOG = LogFactory.getLog(getClass()); - protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - private final static byte[] FAMILY = Bytes.toBytes("cf"); - - private byte[] emptySnapshotName; - private byte[] snapshotName; - private int tableNumFiles; - private TableName tableName; - private Admin admin; - public static void setUpBaseConf(Configuration conf) { - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.setInt("hbase.regionserver.msginterval", 100); - conf.setInt("hbase.client.pause", 250); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - conf.setBoolean("hbase.master.enabletable.roundrobin", true); - conf.setInt("mapreduce.map.maxattempts", 10); + TestExportSnapshot.setUpBaseConf(conf); conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); } @@ -91,342 +52,13 @@ public class TestMobExportSnapshot { TEST_UTIL.startMiniMapReduceCluster(); } - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); - TEST_UTIL.shutdownMiniCluster(); - } - - /** - * Create a table and take a snapshot of the table used by the export test. - */ - @Before - public void setUp() throws Exception { - this.admin = TEST_UTIL.getHBaseAdmin(); - - long tid = System.currentTimeMillis(); - tableName = TableName.valueOf("testtb-" + tid); - snapshotName = Bytes.toBytes("snaptb0-" + tid); - emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + tid); - - // create Table + @Override + protected void createTable() throws Exception { MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY); - - // Take an empty snapshot - admin.snapshot(emptySnapshotName, tableName); - - // Add some rows - SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); - tableNumFiles = admin.getTableRegions(tableName).size(); - - // take a snapshot - admin.snapshot(snapshotName, tableName); - } - - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(tableName); - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - } - - /** - * Verfy the result of getBalanceSplits() method. - * The result are groups of files, used as input list for the "export" mappers. - * All the groups should have similar amount of data. - * - * The input list is a pair of file path and length. - * The getBalanceSplits() function sort it by length, - * and assign to each group a file, going back and forth through the groups. - */ - @Test - public void testBalanceSplit() throws Exception { - // Create a list of files - List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>(); - for (long i = 0; i <= 20; i++) { - SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder() - .setType(SnapshotFileInfo.Type.HFILE) - .setHfile("file-" + i) - .build(); - files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, i)); - } - - // Create 5 groups (total size 210) - // group 0: 20, 11, 10, 1 (total size: 42) - // group 1: 19, 12, 9, 2 (total size: 42) - // group 2: 18, 13, 8, 3 (total size: 42) - // group 3: 17, 12, 7, 4 (total size: 42) - // group 4: 16, 11, 6, 5 (total size: 42) - List<List<Pair<SnapshotFileInfo, Long>>> splits = ExportSnapshot.getBalancedSplits(files, 5); - assertEquals(5, splits.size()); - - String[] split0 = new String[] {"file-20", "file-11", "file-10", "file-1", "file-0"}; - verifyBalanceSplit(splits.get(0), split0, 42); - String[] split1 = new String[] {"file-19", "file-12", "file-9", "file-2"}; - verifyBalanceSplit(splits.get(1), split1, 42); - String[] split2 = new String[] {"file-18", "file-13", "file-8", "file-3"}; - verifyBalanceSplit(splits.get(2), split2, 42); - String[] split3 = new String[] {"file-17", "file-14", "file-7", "file-4"}; - verifyBalanceSplit(splits.get(3), split3, 42); - String[] split4 = new String[] {"file-16", "file-15", "file-6", "file-5"}; - verifyBalanceSplit(splits.get(4), split4, 42); - } - - private void verifyBalanceSplit(final List<Pair<SnapshotFileInfo, Long>> split, - final String[] expected, final long expectedSize) { - assertEquals(expected.length, split.size()); - long totalSize = 0; - for (int i = 0; i < expected.length; ++i) { - Pair<SnapshotFileInfo, Long> fileInfo = split.get(i); - assertEquals(expected[i], fileInfo.getFirst().getHfile()); - totalSize += fileInfo.getSecond(); - } - assertEquals(expectedSize, totalSize); - } - - /** - * Verify if exported snapshot and copied files matches the original one. - */ - @Test - public void testExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } - - @Test - public void testExportFileSystemStateWithSkipTmp() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); - } - - @Test - public void testEmptyExportFileSystemState() throws Exception { - testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); - } - - @Test - public void testConsecutiveExports() throws Exception { - Path copyDir = getLocalDestinationDir(); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); - testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); - removeExportDir(copyDir); - } - - @Test - public void testExportWithTargetName() throws Exception { - final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); - testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); - } - - /** - * Mock a snapshot with files in the archive dir, - * two regions, and one reference file. - */ - @Test - public void testSnapshotWithRefsExportFileSystemState() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - - Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - FileSystem fs = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("tableWithRefsV1"); - testSnapshotWithRefsExportFileSystemState(builder); - - snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - builder = snapshotMock.createSnapshotV2("tableWithRefsV2"); - testSnapshotWithRefsExportFileSystemState(builder); - } - - /** - * Generates a couple of regions for the specified SnapshotMock, - * and then it will run the export and verification. - */ - private void testSnapshotWithRefsExportFileSystemState(SnapshotMock.SnapshotBuilder builder) - throws Exception { - Path[] r1Files = builder.addRegion(); - Path[] r2Files = builder.addRegion(); - builder.commit(); - int snapshotFilesCount = r1Files.length + r2Files.length; - - byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName()); - TableName tableName = builder.getTableDescriptor().getTableName(); - testExportFileSystemState(tableName, snapshotName, snapshotName, snapshotFilesCount); - } - - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected) throws Exception { - Path copyDir = getHdfsDestinationDir(); - testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir, false); - removeExportDir(copyDir); - } - - /** - * Test ExportSnapshot - */ - private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, - final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) - throws Exception { - URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - - List<String> opts = new ArrayList<String>(); - opts.add("-snapshot"); - opts.add(Bytes.toString(snapshotName)); - opts.add("-copy-to"); - opts.add(copyDir.toString()); - if (targetName != snapshotName) { - opts.add("-target"); - opts.add(Bytes.toString(targetName)); - } - if (overwrite) opts.add("-overwrite"); - - // Export Snapshot - int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), - opts.toArray(new String[opts.size()])); - assertEquals(0, res); - - // Verify File-System state - FileStatus[] rootFiles = fs.listStatus(copyDir); - assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length); - for (FileStatus fileStatus: rootFiles) { - String name = fileStatus.getPath().getName(); - assertTrue(fileStatus.isDirectory()); - assertTrue(name.equals(HConstants.SNAPSHOT_DIR_NAME) || - name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY)); - } - - // compare the snapshot metadata and verify the hfiles - final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration()); - final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); - final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); - verifySnapshotDir(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir), - fs, new Path(copyDir, targetDir)); - Set<String> snapshotFiles = verifySnapshot(fs, copyDir, tableName, Bytes.toString(targetName)); - assertEquals(filesExpected, snapshotFiles.size()); - } - - /** - * Check that ExportSnapshot will return a failure if something fails. - */ - @Test - public void testExportFailure() throws Exception { - assertEquals(1, runExportAndInjectFailures(snapshotName, false)); - } - - /** - * Check that ExportSnapshot will succede if something fails but the retry succede. - */ - @Test - public void testExportRetry() throws Exception { - assertEquals(0, runExportAndInjectFailures(snapshotName, true)); - } - - /* - * Execute the ExportSnapshot job injecting failures - */ - private int runExportAndInjectFailures(final byte[] snapshotName, boolean retry) - throws Exception { - Path copyDir = getLocalDestinationDir(); - URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri(); - FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); - copyDir = copyDir.makeQualified(fs); - - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.setBoolean(ExportSnapshot.CONF_TEST_FAILURE, true); - conf.setBoolean(ExportSnapshot.CONF_TEST_RETRY, retry); - - // Export Snapshot - Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - int res = ExportSnapshot.innerMain(conf, new String[] { - "-snapshot", Bytes.toString(snapshotName), - "-copy-from", sourceDir.toString(), - "-copy-to", copyDir.toString() - }); - return res; - } - - /* - * verify if the snapshot folder on file-system 1 match the one on file-system 2 - */ - private void verifySnapshotDir(final FileSystem fs1, final Path root1, - final FileSystem fs2, final Path root2) throws IOException { - assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2)); - } - - /* - * Verify if the files exists - */ - private Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, - final TableName tableName, final String snapshotName) throws IOException { - final Path exportedSnapshot = new Path(rootDir, - new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName)); - final Set<String> snapshotFiles = new HashSet<String>(); - final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); - SnapshotReferenceUtil.visitReferencedFiles(TEST_UTIL.getConfiguration(), fs, exportedSnapshot, - new SnapshotReferenceUtil.SnapshotVisitor() { - @Override - public void storeFile(final HRegionInfo regionInfo, final String family, - final SnapshotRegionManifest.StoreFile storeFile) throws IOException { - if(MobUtils.isMobRegionInfo(regionInfo)) - return; - String hfile = storeFile.getName(); - snapshotFiles.add(hfile); - if (storeFile.hasReference()) { - // Nothing to do here, we have already the reference embedded - } else { - verifyNonEmptyFile(new Path(exportedArchive, - new Path(FSUtils.getTableDir(new Path("./"), tableName), - new Path(regionInfo.getEncodedName(), new Path(family, hfile))))); - } - } - - private void verifyNonEmptyFile(final Path path) throws IOException { - assertTrue(path + " should exists", fs.exists(path)); - assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0); - } - }); - - // Verify Snapshot description - SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot); - assertTrue(desc.getName().equals(snapshotName)); - assertTrue(desc.getTable().equals(tableName.getNameAsString())); - return snapshotFiles; - } - - private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir) - throws IOException { - Set<String> files = new HashSet<String>(); - int rootPrefix = root.toString().length(); - FileStatus[] list = FSUtils.listStatus(fs, dir); - if (list != null) { - for (FileStatus fstat: list) { - LOG.debug(fstat.getPath()); - if (fstat.isDirectory()) { - files.addAll(listFiles(fs, root, fstat.getPath())); - } else { - files.add(fstat.getPath().toString().substring(rootPrefix)); - } - } - } - return files; - } - - private Path getHdfsDestinationDir() { - Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - Path path = new Path(new Path(rootDir, "export-test"), "export-" + System.currentTimeMillis()); - LOG.info("HDFS export destination path: " + path); - return path; - } - - private Path getLocalDestinationDir() { - Path path = TEST_UTIL.getDataTestDir("local-export-" + System.currentTimeMillis()); - LOG.info("Local export destination path: " + path); - return path; } - private void removeExportDir(final Path path) throws IOException { - FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); - fs.delete(path, true); + @Override + protected boolean bypassRegion(HRegionInfo regionInfo) { + return MobUtils.isMobRegionInfo(regionInfo); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java index a8e88b1..eefb180 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobFlushSnapshotFromClient.java @@ -17,44 +17,19 @@ */ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.*; - import java.io.IOException; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.log4j.Level; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -66,482 +41,33 @@ import org.junit.experimental.categories.Category; * because there will be a few more flavors of snapshots that need to run these tests. */ @Category({ClientTests.class, LargeTests.class}) -public class TestMobFlushSnapshotFromClient { +public class TestMobFlushSnapshotFromClient extends TestFlushSnapshotFromClient { private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final int NUM_RS = 2; - private static final String STRING_TABLE_NAME = "test"; - private static final byte[] TEST_FAM = Bytes.toBytes("fam"); - private static final TableName TABLE_NAME = - TableName.valueOf(STRING_TABLE_NAME); - private final int DEFAULT_NUM_ROWS = 100; - /** - * Setup the config for the cluster - * @throws Exception on failure - */ @BeforeClass public static void setupCluster() throws Exception { - ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL); - ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL); setupConf(UTIL.getConfiguration()); - UTIL.startMiniCluster(NUM_RS); + UTIL.startMiniCluster(3); } - private static void setupConf(Configuration conf) { - // disable the ui - conf.setInt("hbase.regionsever.info.port", -1); - // change the flush size to a small amount, regulating number of store files - conf.setInt("hbase.hregion.memstore.flush.size", 25000); - // so make sure we get a compaction when doing a load, but keep around some - // files in the store - conf.setInt("hbase.hstore.compaction.min", 10); - conf.setInt("hbase.hstore.compactionThreshold", 10); - // block writes if we get to 12 store files - conf.setInt("hbase.hstore.blockingStoreFiles", 12); - // Enable snapshot - conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, - ConstantSizeRegionSplitPolicy.class.getName()); - conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); + protected static void setupConf(Configuration conf) { + TestFlushSnapshotFromClient.setupConf(conf); + UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); } - @Before - public void setup() throws Exception { + @Override + protected void createTable() throws Exception { MobSnapshotTestingUtils.createMobTable(UTIL, TABLE_NAME, 1, TEST_FAM); } - @After - public void tearDown() throws Exception { - UTIL.deleteTable(TABLE_NAME); - - SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(UTIL); - } - - @AfterClass - public static void cleanupTest() throws Exception { - try { - UTIL.shutdownMiniCluster(); - } catch (Exception e) { - LOG.warn("failure shutting down cluster", e); - } - } - - /** - * Test simple flush snapshotting a table that is online - * @throws Exception - */ - @Test (timeout=300000) - public void testFlushTableSnapshot() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - - // put some stuff in the table - Table table = ConnectionFactory.createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME); - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); - - LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - // take a snapshot of the enabled table - String snapshotString = "offlineTableSnapshot"; - byte[] snapshot = Bytes.toBytes(snapshotString); - admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH); - LOG.debug("Snapshot completed."); - - // make sure we have the snapshot - List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); - - // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); - } - - /** - * Test snapshotting a table that is online without flushing - * @throws Exception - */ - @Test(timeout=30000) - public void testSkipFlushTableSnapshot() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - - // put some stuff in the table - Table table = ConnectionFactory.createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME); - UTIL.loadTable(table, TEST_FAM); - - LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - // take a snapshot of the enabled table - String snapshotString = "skipFlushTableSnapshot"; - byte[] snapshot = Bytes.toBytes(snapshotString); - admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH); - LOG.debug("Snapshot completed."); - - // make sure we have the snapshot - List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); - - // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); - - admin.deleteSnapshot(snapshot); - snapshots = admin.listSnapshots(); - SnapshotTestingUtils.assertNoSnapshots(admin); - } - - - /** - * Test simple flush snapshotting a table that is online - * @throws Exception - */ - @Test (timeout=300000) - public void testFlushTableSnapshotWithProcedure() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - - // put some stuff in the table - Table table = ConnectionFactory.createConnection(UTIL.getConfiguration()).getTable(TABLE_NAME); - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); - - LOG.debug("FS state before snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - // take a snapshot of the enabled table - String snapshotString = "offlineTableSnapshot"; - byte[] snapshot = Bytes.toBytes(snapshotString); - Map<String, String> props = new HashMap<String, String>(); - props.put("table", TABLE_NAME.getNameAsString()); - admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, - snapshotString, props); - - - LOG.debug("Snapshot completed."); - - // make sure we have the snapshot - List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, - snapshot, TABLE_NAME); - - // make sure its a valid snapshot - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - LOG.debug("FS state after snapshot:"); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - - SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir, - admin, fs); - } - - @Test (timeout=300000) - public void testSnapshotFailsOnNonExistantTable() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - TableName tableName = TableName.valueOf("_not_a_table"); - - // make sure the table doesn't exist - boolean fail = false; - do { - try { - admin.getTableDescriptor(tableName); - fail = true; - LOG.error("Table:" + tableName + " already exists, checking a new name"); - tableName = TableName.valueOf(tableName+"!"); - } catch (TableNotFoundException e) { - fail = false; - } - } while (fail); - - // snapshot the non-existant table - try { - admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH); - fail("Snapshot succeeded even though there is not table."); - } catch (SnapshotCreationException e) { - LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage()); - } - } - - @Test(timeout = 300000) - public void testAsyncFlushSnapshot() throws Exception { - Admin admin = UTIL.getHBaseAdmin(); - SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot") - .setTable(TABLE_NAME.getNameAsString()) - .setType(SnapshotDescription.Type.FLUSH) - .build(); - - // take the snapshot async - admin.takeSnapshotAsync(snapshot); - - // constantly loop, looking for the snapshot to complete - HMaster master = UTIL.getMiniHBaseCluster().getMaster(); - SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200); - LOG.info(" === Async Snapshot Completed ==="); - FSUtils.logFileSystemState(UTIL.getTestFileSystem(), - FSUtils.getRootDir(UTIL.getConfiguration()), LOG); - // make sure we get the snapshot - SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot); - } - - @Test (timeout=300000) - public void testSnapshotStateAfterMerge() throws Exception { - int numRows = DEFAULT_NUM_ROWS; - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - // load the table so we have some data - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); - - // Take a snapshot - String snapshotBeforeMergeName = "snapshotBeforeMerge"; - admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH); - - // Clone the table - TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge"); - admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName); - SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName); - - // Merge two regions - List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME); - Collections.sort(regions, new Comparator<HRegionInfo>() { - public int compare(HRegionInfo r1, HRegionInfo r2) { - return Bytes.compareTo(r1.getStartKey(), r2.getStartKey()); - } - }); - - int numRegions = admin.getTableRegions(TABLE_NAME).size(); - int numRegionsAfterMerge = numRegions - 2; - admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), - regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), - regions.get(6).getEncodedNameAsBytes(), true); - - // Verify that there's one region less - waitRegionsAfterMerge(numRegionsAfterMerge); - assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size()); - - // Clone the table - TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge"); - admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName); - SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName); - - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneBeforeMergeName, numRows); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneAfterMergeName, numRows); - - // test that we can delete the snapshot - UTIL.deleteTable(cloneAfterMergeName); - UTIL.deleteTable(cloneBeforeMergeName); - } - - @Test (timeout=300000) - public void testTakeSnapshotAfterMerge() throws Exception { - int numRows = DEFAULT_NUM_ROWS; - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - // load the table so we have some data - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); - - // Merge two regions - List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME); - Collections.sort(regions, new Comparator<HRegionInfo>() { - public int compare(HRegionInfo r1, HRegionInfo r2) { - return Bytes.compareTo(r1.getStartKey(), r2.getStartKey()); - } - }); - - int numRegions = admin.getTableRegions(TABLE_NAME).size(); - int numRegionsAfterMerge = numRegions - 2; - admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(), - regions.get(2).getEncodedNameAsBytes(), true); - admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(), - regions.get(6).getEncodedNameAsBytes(), true); - - waitRegionsAfterMerge(numRegionsAfterMerge); - assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size()); - - // Take a snapshot - String snapshotName = "snapshotAfterMerge"; - SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(), - SnapshotDescription.Type.FLUSH, 3); - - // Clone the table - TableName cloneName = TableName.valueOf("cloneMerge"); - admin.cloneSnapshot(snapshotName, cloneName); - SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName); - - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, TABLE_NAME, numRows); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, cloneName, numRows); - - // test that we can delete the snapshot - UTIL.deleteTable(cloneName); - } - - /** - * Basic end-to-end test of simple-flush-based snapshots - */ - @Test (timeout=300000) - public void testFlushCreateListDestroy() throws Exception { - LOG.debug("------- Starting Snapshot test -------------"); - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - // load the table so we have some data - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); - - String snapshotName = "flushSnapshotCreateListDestroy"; - FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); - Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); - SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM), - snapshotName, rootDir, fs, true); - } - - /** - * Demonstrate that we reject snapshot requests if there is a snapshot already running on the - * same table currently running and that concurrent snapshots on different tables can both - * succeed concurrently. - */ - @Test(timeout=300000) - public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException { - final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2"; - final TableName TABLE2_NAME = - TableName.valueOf(STRING_TABLE2_NAME); - - int ssNum = 20; - Admin admin = UTIL.getHBaseAdmin(); - // make sure we don't fail on listing snapshots - SnapshotTestingUtils.assertNoSnapshots(admin); - // create second testing table - SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM); - // load the table so we have some data - SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); - SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM); - - final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum); - // We'll have one of these per thread - class SSRunnable implements Runnable { - SnapshotDescription ss; - SSRunnable(SnapshotDescription ss) { - this.ss = ss; - } - - @Override - public void run() { - try { - Admin admin = UTIL.getHBaseAdmin(); - LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); - admin.takeSnapshotAsync(ss); - } catch (Exception e) { - LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString( - ss) - + ". This is ok, we expect some", e); - } - LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss)); - toBeSubmitted.countDown(); - } - }; - - // build descriptions - SnapshotDescription[] descs = new SnapshotDescription[ssNum]; - for (int i = 0; i < ssNum; i++) { - SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); - builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString()); - builder.setName("ss"+i); - builder.setType(SnapshotDescription.Type.FLUSH); - descs[i] = builder.build(); - } - - // kick each off its own thread - for (int i=0 ; i < ssNum; i++) { - new Thread(new SSRunnable(descs[i])).start(); - } - - // wait until all have been submitted - toBeSubmitted.await(); - - // loop until all are done. - while (true) { - int doneCount = 0; - for (SnapshotDescription ss : descs) { - try { - if (admin.isSnapshotFinished(ss)) { - doneCount++; - } - } catch (Exception e) { - LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e); - doneCount++; - } - } - if (doneCount == descs.length) { - break; - } - Thread.sleep(100); - } - - // dump for debugging - logFSTree(FSUtils.getRootDir(UTIL.getConfiguration())); - - List<SnapshotDescription> taken = admin.listSnapshots(); - int takenSize = taken.size(); - LOG.info("Taken " + takenSize + " snapshots: " + taken); - assertTrue("We expect at least 1 request to be rejected because of we concurrently" + - " issued many requests", takenSize < ssNum && takenSize > 0); - - // Verify that there's at least one snapshot per table - int t1SnapshotsCount = 0; - int t2SnapshotsCount = 0; - for (SnapshotDescription ss : taken) { - if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) { - t1SnapshotsCount++; - } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) { - t2SnapshotsCount++; - } - } - assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0); - assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0); - - UTIL.deleteTable(TABLE2_NAME); - } - - private void logFSTree(Path root) throws IOException { - FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG); + @Override + protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, + long expectedRows) throws IOException { + MobSnapshotTestingUtils.verifyMobRowCount(util, tableName, expectedRows); } - private void waitRegionsAfterMerge(final long numRegionsAfterMerge) - throws IOException, InterruptedException { - Admin admin = UTIL.getHBaseAdmin(); - // Verify that there's one region less - long startTime = System.currentTimeMillis(); - while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) { - // This may be flaky... if after 15sec the merge is not complete give up - // it will fail in the assertEquals(numRegionsAfterMerge). - if ((System.currentTimeMillis() - startTime) > 15000) - break; - Thread.sleep(100); - } - SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME); + @Override + protected int countRows(final Table table, final byte[]... families) throws IOException { + return MobSnapshotTestingUtils.countMobRows(table, families); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java index cb58b17..6893346 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreFlushSnapshotFromClient.java @@ -21,26 +21,15 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.master.MasterFileSystem; -import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Test; import org.junit.experimental.categories.Category; /** @@ -50,162 +39,33 @@ import org.junit.experimental.categories.Category; * this because there will be a few more flavors of snapshots that need to run these tests. */ @Category({ClientTests.class,LargeTests.class}) -public class TestMobRestoreFlushSnapshotFromClient { +public class TestMobRestoreFlushSnapshotFromClient extends TestRestoreFlushSnapshotFromClient { final Log LOG = LogFactory.getLog(getClass()); - private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - private final byte[] FAMILY = Bytes.toBytes("cf"); - - private byte[] snapshotName0; - private byte[] snapshotName1; - private byte[] snapshotName2; - private int snapshot0Rows; - private int snapshot1Rows; - private TableName tableName; - private Admin admin; - @BeforeClass - public static void setUpBeforeClass() throws Exception { - UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); - UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); - UTIL.getConfiguration().setInt("hbase.client.pause", 250); - UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); - UTIL.getConfiguration().setBoolean( - "hbase.master.enabletable.roundrobin", true); - - // Enable snapshot - UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); - UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY, - RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2); - - UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); UTIL.startMiniCluster(3); } - @AfterClass - public static void tearDownAfterClass() throws Exception { - UTIL.shutdownMiniCluster(); + protected static void setupConf(Configuration conf) { + TestRestoreFlushSnapshotFromClient.setupConf(conf); + UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); } - /** - * Initialize the tests with a table filled with some data - * and two snapshots (snapshotName0, snapshotName1) of different states. - * The tableName, snapshotNames and the number of rows in the snapshot are initialized. - */ - @Before - public void setup() throws Exception { - this.admin = UTIL.getHBaseAdmin(); - - long tid = System.currentTimeMillis(); - tableName = TableName.valueOf("testtb-" + tid); - snapshotName0 = Bytes.toBytes("snaptb0-" + tid); - snapshotName1 = Bytes.toBytes("snaptb1-" + tid); - snapshotName2 = Bytes.toBytes("snaptb2-" + tid); - - // create Table + @Override + protected void createTable() throws Exception { MobSnapshotTestingUtils.createMobTable(UTIL, tableName, 1, FAMILY); - - Table table = ConnectionFactory.createConnection(UTIL.getConfiguration()).getTable(tableName); - SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY); - snapshot0Rows = MobSnapshotTestingUtils.countMobRows(table); - LOG.info("=== before snapshot with 500 rows"); - logFSTree(); - - // take a snapshot - admin.snapshot(Bytes.toString(snapshotName0), tableName, - SnapshotDescription.Type.FLUSH); - - LOG.info("=== after snapshot with 500 rows"); - logFSTree(); - - // insert more data - SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY); - snapshot1Rows = MobSnapshotTestingUtils.countMobRows(table); - LOG.info("=== before snapshot with 1000 rows"); - logFSTree(); - - // take a snapshot of the updated table - admin.snapshot(Bytes.toString(snapshotName1), tableName, - SnapshotDescription.Type.FLUSH); - LOG.info("=== after snapshot with 1000 rows"); - logFSTree(); - table.close(); } - @After - public void tearDown() throws Exception { - SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(UTIL); - } - - @Test - public void testTakeFlushSnapshot() throws IOException { - // taking happens in setup. - } - - @Test - public void testRestoreSnapshot() throws IOException { - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot1Rows); - - // Restore from snapshot-0 - admin.disableTable(tableName); - admin.restoreSnapshot(snapshotName0); - logFSTree(); - admin.enableTable(tableName); - LOG.info("=== after restore with 500 row snapshot"); - logFSTree(); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot0Rows); - - // Restore from snapshot-1 - admin.disableTable(tableName); - admin.restoreSnapshot(snapshotName1); - admin.enableTable(tableName); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshot1Rows); - } - - @Test(expected=SnapshotDoesNotExistException.class) - public void testCloneNonExistentSnapshot() throws IOException, InterruptedException { - String snapshotName = "random-snapshot-" + System.currentTimeMillis(); - TableName tableName = TableName.valueOf("random-table-" + System.currentTimeMillis()); - admin.cloneSnapshot(snapshotName, tableName); - } - - @Test - public void testCloneSnapshot() throws IOException, InterruptedException { - TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis()); - testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows); - testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows); - } - - private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName, - int snapshotRows) throws IOException, InterruptedException { - // create a new table from snapshot - admin.cloneSnapshot(snapshotName, tableName); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, tableName, snapshotRows); - - UTIL.deleteTable(tableName); - } - - @Test - public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException { - TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis()); - admin.cloneSnapshot(snapshotName0, clonedTableName); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, clonedTableName, snapshot0Rows); - admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH); - UTIL.deleteTable(clonedTableName); - - admin.cloneSnapshot(snapshotName2, clonedTableName); - MobSnapshotTestingUtils.verifyMobRowCount(UTIL, clonedTableName, snapshot0Rows); - UTIL.deleteTable(clonedTableName); + @Override + protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, + long expectedRows) throws IOException { + MobSnapshotTestingUtils.verifyMobRowCount(util, tableName, expectedRows); } - // ========================================================================== - // Helpers - // ========================================================================== - private void logFSTree() throws IOException { - MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); - FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG); + @Override + protected int countRows(final Table table, final byte[]... families) throws IOException { + return MobSnapshotTestingUtils.countMobRows(table, families); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java index c5ec3ad..9934513 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobRestoreSnapshotHelper.java @@ -17,31 +17,14 @@ */ package org.apache.hadoop.hbase.snapshot; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.testclassification.SmallTests; -import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; -import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.mob.MobConstants; -import org.apache.hadoop.hbase.monitoring.MonitoredTask; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils.SnapshotMock; -import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -49,112 +32,16 @@ import org.mockito.Mockito; * Test the restore/clone operation from a file-system point of view. */ @Category(SmallTests.class) -public class TestMobRestoreSnapshotHelper { +public class TestMobRestoreSnapshotHelper extends TestRestoreSnapshotHelper { final Log LOG = LogFactory.getLog(getClass()); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - private Configuration conf; - private FileSystem fs; - private Path rootDir; - - @Before - public void setup() throws Exception { - rootDir = TEST_UTIL.getDataTestDir("testRestore"); - fs = TEST_UTIL.getTestFileSystem(); - TEST_UTIL.getConfiguration().setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); - conf = TEST_UTIL.getConfiguration(); - FSUtils.setRootDir(conf, rootDir); - } - - @After - public void tearDown() throws Exception { - fs.delete(TEST_UTIL.getDataTestDir(), true); - } - - @Test - public void testRestore() throws IOException { - // Test Rolling-Upgrade like Snapshot. - // half machines writing using v1 and the others using v2 format. - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); - SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot"); - builder.addRegionV1(); - builder.addRegionV2(); - builder.addRegionV2(); - builder.addRegionV1(); - Path snapshotDir = builder.commit(); - HTableDescriptor htd = builder.getTableDescriptor(); - SnapshotDescription desc = builder.getSnapshotDescription(); - - // Test clone a snapshot - HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone"); - testRestore(snapshotDir, desc, htdClone); - verifyRestore(rootDir, htd, htdClone); - - // Test clone a clone ("link to link") - SnapshotDescription cloneDesc = SnapshotDescription.newBuilder() - .setName("cloneSnapshot") - .setTable("testtb-clone") - .build(); - Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName()); - HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2"); - testRestore(cloneDir, cloneDesc, htdClone2); - verifyRestore(rootDir, htd, htdClone2); - } - - private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd, - final HTableDescriptor htdClone) throws IOException { - String[] files = SnapshotTestingUtils.listHFileNames(fs, - FSUtils.getTableDir(rootDir, htdClone.getTableName())); - assertEquals(12, files.length); - for (int i = 0; i < files.length; i += 2) { - String linkFile = files[i]; - String refFile = files[i+1]; - assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile)); - assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile)); - assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile)); - Path refPath = getReferredToFile(refFile); - LOG.debug("get reference name for file " + refFile + " = " + refPath); - assertTrue(refPath.getName() + " should be a HFileLink", - HFileLink.isHFileLink(refPath.getName())); - assertEquals(linkFile, refPath.getName()); - } - } - - /** - * Execute the restore operation - * @param snapshotDir The snapshot directory to use as "restore source" - * @param sd The snapshot descriptor - * @param htdClone The HTableDescriptor of the table to restore/clone. - */ - public void testRestore(final Path snapshotDir, final SnapshotDescription sd, - final HTableDescriptor htdClone) throws IOException { - LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); - FSUtils.logFileSystemState(fs, rootDir, LOG); - - new FSTableDescriptors(conf).createTableDescriptor(htdClone); - RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); - helper.restoreHdfsRegions(); - - LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); - FSUtils.logFileSystemState(fs, rootDir, LOG); - } - - /** - * Initialize the restore helper, based on the snapshot and table information provided. - */ - private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, - final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException { - ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); - MonitoredTask status = Mockito.mock(MonitoredTask.class); - - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); - return new RestoreSnapshotHelper(conf, fs, manifest, - htdClone, rootDir, monitor, status); + @Override + protected void setupConf(Configuration conf) { + conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); } - private Path getReferredToFile(final String referenceName) { - Path fakeBasePath = new Path(new Path("table", "region"), "cf"); - return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName)); + @Override + protected SnapshotMock createSnapshotMock() throws IOException { + return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java index 219e694..99ece6c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreFlushSnapshotFromClient.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; @@ -51,20 +52,25 @@ import org.junit.experimental.categories.Category; public class TestRestoreFlushSnapshotFromClient { private static final Log LOG = LogFactory.getLog(TestRestoreFlushSnapshotFromClient.class); - private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); + protected final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private final byte[] FAMILY = Bytes.toBytes("cf"); + protected final byte[] FAMILY = Bytes.toBytes("cf"); - private byte[] snapshotName0; - private byte[] snapshotName1; - private byte[] snapshotName2; - private int snapshot0Rows; - private int snapshot1Rows; - private TableName tableName; - private Admin admin; + protected byte[] snapshotName0; + protected byte[] snapshotName1; + protected byte[] snapshotName2; + protected int snapshot0Rows; + protected int snapshot1Rows; + protected TableName tableName; + protected Admin admin; @BeforeClass - public static void setUpBeforeClass() throws Exception { + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(3); + } + + protected static void setupConf(Configuration conf) { UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true); UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); UTIL.getConfiguration().setInt("hbase.client.pause", 250); @@ -76,8 +82,6 @@ public class TestRestoreFlushSnapshotFromClient { UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY, RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2); - - UTIL.startMiniCluster(3); } @AfterClass @@ -85,6 +89,10 @@ public class TestRestoreFlushSnapshotFromClient { UTIL.shutdownMiniCluster(); } + protected void createTable() throws Exception { + SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY); + } + /** * Initialize the tests with a table filled with some data * and two snapshots (snapshotName0, snapshotName1) of different states. @@ -101,10 +109,10 @@ public class TestRestoreFlushSnapshotFromClient { snapshotName2 = Bytes.toBytes("snaptb2-" + tid); // create Table and disable it - SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY); + createTable(); SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY); Table table = UTIL.getConnection().getTable(tableName); - snapshot0Rows = UTIL.countRows(table); + snapshot0Rows = countRows(table); LOG.info("=== before snapshot with 500 rows"); logFSTree(); @@ -117,7 +125,7 @@ public class TestRestoreFlushSnapshotFromClient { // insert more data SnapshotTestingUtils.loadData(UTIL, tableName, 500, FAMILY); - snapshot1Rows = UTIL.countRows(table); + snapshot1Rows = countRows(table); LOG.info("=== before snapshot with 1000 rows"); logFSTree(); @@ -142,7 +150,7 @@ public class TestRestoreFlushSnapshotFromClient { @Test public void testRestoreSnapshot() throws IOException { - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); + verifyRowCount(UTIL, tableName, snapshot1Rows); // Restore from snapshot-0 admin.disableTable(tableName); @@ -151,13 +159,13 @@ public class TestRestoreFlushSnapshotFromClient { admin.enableTable(tableName); LOG.info("=== after restore with 500 row snapshot"); logFSTree(); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows); + verifyRowCount(UTIL, tableName, snapshot0Rows); // Restore from snapshot-1 admin.disableTable(tableName); admin.restoreSnapshot(snapshotName1); admin.enableTable(tableName); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); + verifyRowCount(UTIL, tableName, snapshot1Rows); } @Test(expected=SnapshotDoesNotExistException.class) @@ -178,7 +186,7 @@ public class TestRestoreFlushSnapshotFromClient { int snapshotRows) throws IOException, InterruptedException { // create a new table from snapshot admin.cloneSnapshot(snapshotName, tableName); - SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshotRows); + verifyRowCount(UTIL, tableName, snapshotRows); UTIL.deleteTable(tableName); } @@ -187,12 +195,12 @@ public class TestRestoreFlushSnapshotFromClient { public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException { TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis()); admin.cloneSnapshot(snapshotName0, clonedTableName); - SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows); + verifyRowCount(UTIL, clonedTableName, snapshot0Rows); admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH); UTIL.deleteTable(clonedTableName); admin.cloneSnapshot(snapshotName2, clonedTableName); - SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows); + verifyRowCount(UTIL, clonedTableName, snapshot0Rows); UTIL.deleteTable(clonedTableName); } @@ -203,4 +211,13 @@ public class TestRestoreFlushSnapshotFromClient { MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG); } + + protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName, + long expectedRows) throws IOException { + SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows); + } + + protected int countRows(final Table table, final byte[]... families) throws IOException { + return UTIL.countRows(table, families); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/5e26ae04/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java index 3fac2fd..365d56c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -53,13 +53,16 @@ import org.mockito.Mockito; public class TestRestoreSnapshotHelper { private static final Log LOG = LogFactory.getLog(TestRestoreSnapshotHelper.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private final static String TEST_HFILE = "abc"; + protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + protected final static String TEST_HFILE = "abc"; - private Configuration conf; - private Path archiveDir; - private FileSystem fs; - private Path rootDir; + protected Configuration conf; + protected Path archiveDir; + protected FileSystem fs; + protected Path rootDir; + + protected void setupConf(Configuration conf) { + } @Before public void setup() throws Exception { @@ -67,6 +70,7 @@ public class TestRestoreSnapshotHelper { archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); fs = TEST_UTIL.getTestFileSystem(); conf = TEST_UTIL.getConfiguration(); + setupConf(conf); FSUtils.setRootDir(conf, rootDir); } @@ -75,11 +79,15 @@ public class TestRestoreSnapshotHelper { fs.delete(TEST_UTIL.getDataTestDir(), true); } + protected SnapshotMock createSnapshotMock() throws IOException { + return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + } + @Test public void testRestore() throws IOException { // Test Rolling-Upgrade like Snapshot. // half machines writing using v1 and the others using v2 format. - SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir); + SnapshotMock snapshotMock = createSnapshotMock(); SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot"); builder.addRegionV1(); builder.addRegionV2(); @@ -129,7 +137,7 @@ public class TestRestoreSnapshotHelper { * @param sd The snapshot descriptor * @param htdClone The HTableDescriptor of the table to restore/clone. */ - public void testRestore(final Path snapshotDir, final SnapshotDescription sd, + private void testRestore(final Path snapshotDir, final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException { LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG);
