http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java deleted file mode 100644 index fe992b6..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestNamespaceUpgrade.java +++ /dev/null @@ -1,354 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.migration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.FsShell; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -import org.apache.hadoop.hbase.security.access.AccessControlLists; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSTableDescriptors; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.util.ToolRunner; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test upgrade from no namespace in 0.94 to namespace directory structure. - * Mainly tests that tables are migrated and consistent. Also verifies - * that snapshots have been migrated correctly. - * - * <p>Uses a tarball which is an image of an 0.94 hbase.rootdir. - * - * <p>Contains tables with currentKeys as the stored keys: - * foo, ns1.foo, ns2.foo - * - * <p>Contains snapshots with snapshot{num}Keys as the contents: - * snapshot1Keys, snapshot2Keys - * - * Image also contains _acl_ table with one region and two storefiles. - * This is needed to test the acl table migration. - * - */ -@Category(MediumTests.class) -public class TestNamespaceUpgrade { - static final Log LOG = LogFactory.getLog(TestNamespaceUpgrade.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private final static String snapshot1Keys[] = - {"1","10","2","3","4","5","6","7","8","9"}; - private final static String snapshot2Keys[] = - {"1","2","3","4","5","6","7","8","9"}; - private final static String currentKeys[] = - {"1","2","3","4","5","6","7","8","9","A"}; - private final static TableName tables[] = - {TableName.valueOf("foo"), TableName.valueOf("ns1.foo"), TableName.valueOf("ns.two.foo")}; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Start up our mini cluster on top of an 0.94 root.dir that has data from - // a 0.94 hbase run and see if we can migrate to 0.96 - TEST_UTIL.startMiniZKCluster(); - TEST_UTIL.startMiniDFSCluster(1); - Path testdir = TEST_UTIL.getDataTestDir("TestNamespaceUpgrade"); - // Untar our test dir. - File untar = untar(new File(testdir.toString())); - // Now copy the untar up into hdfs so when we start hbase, we'll run from it. - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setBoolean("hbase.assignment.usezk", true); - FsShell shell = new FsShell(conf); - FileSystem fs = FileSystem.get(conf); - // find where hbase will root itself, so we can copy filesystem there - Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath(); - if (!fs.isDirectory(hbaseRootDir.getParent())) { - // mkdir at first - fs.mkdirs(hbaseRootDir.getParent()); - } - if(org.apache.hadoop.util.VersionInfo.getVersion().startsWith("2.")) { - LOG.info("Hadoop version is 2.x, pre-migrating snapshot dir"); - FileSystem localFS = FileSystem.getLocal(conf); - if(!localFS.rename(new Path(untar.toString(), HConstants.OLD_SNAPSHOT_DIR_NAME), - new Path(untar.toString(), HConstants.SNAPSHOT_DIR_NAME))) { - throw new IllegalStateException("Failed to move snapshot dir to 2.x expectation"); - } - } - doFsCommand(shell, - new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()}); - doFsCommand(shell, new String [] {"-lsr", "/"}); - // See whats in minihdfs. - Configuration toolConf = TEST_UTIL.getConfiguration(); - conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDefaultRootDirPath().toString()); - ToolRunner.run(toolConf, new NamespaceUpgrade(), new String[]{"--upgrade"}); - assertTrue(FSUtils.getVersion(fs, hbaseRootDir).equals(HConstants.FILE_SYSTEM_VERSION)); - doFsCommand(shell, new String [] {"-lsr", "/"}); - TEST_UTIL.startMiniHBaseCluster(1, 1); - - for(TableName table: tables) { - int count = 0; - for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new Scan())) { - assertEquals(currentKeys[count++], Bytes.toString(res.getRow())); - } - Assert.assertEquals(currentKeys.length, count); - } - assertEquals(2, TEST_UTIL.getHBaseAdmin().listNamespaceDescriptors().length); - - //verify ACL table is migrated - HTable secureTable = new HTable(conf, AccessControlLists.ACL_TABLE_NAME); - ResultScanner scanner = secureTable.getScanner(new Scan()); - int count = 0; - for(Result r : scanner) { - count++; - } - assertEquals(3, count); - assertFalse(TEST_UTIL.getHBaseAdmin().tableExists(TableName.valueOf("_acl_"))); - - //verify ACL table was compacted - List<HRegion> regions = TEST_UTIL.getMiniHBaseCluster().getRegions(secureTable.getName()); - for(HRegion region : regions) { - assertEquals(1, region.getStores().size()); - } - } - - static File untar(final File testdir) throws IOException { - // Find the src data under src/test/data - final String datafile = "TestNamespaceUpgrade"; - File srcTarFile = new File( - System.getProperty("project.build.testSourceDirectory", "src/test") + - File.separator + "data" + File.separator + datafile + ".tgz"); - File homedir = new File(testdir.toString()); - File tgtUntarDir = new File(homedir, "hbase"); - if (tgtUntarDir.exists()) { - if (!FileUtil.fullyDelete(tgtUntarDir)) { - throw new IOException("Failed delete of " + tgtUntarDir.toString()); - } - } - if (!srcTarFile.exists()) { - throw new IOException(srcTarFile+" does not exist"); - } - LOG.info("Untarring " + srcTarFile + " into " + homedir.toString()); - FileUtil.unTar(srcTarFile, homedir); - Assert.assertTrue(tgtUntarDir.exists()); - return tgtUntarDir; - } - - private static void doFsCommand(final FsShell shell, final String [] args) - throws Exception { - // Run the 'put' command. - int errcode = shell.run(args); - if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode); - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Test (timeout=300000) - public void testSnapshots() throws IOException, InterruptedException { - String snapshots[][] = {snapshot1Keys, snapshot2Keys}; - for(int i = 1; i <= snapshots.length; i++) { - for(TableName table: tables) { - TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot"+i, TableName.valueOf(table+"_clone"+i)); - FSUtils.logFileSystemState(FileSystem.get(TEST_UTIL.getConfiguration()), - FSUtils.getRootDir(TEST_UTIL.getConfiguration()), - LOG); - int count = 0; - for(Result res: new HTable(TEST_UTIL.getConfiguration(), table+"_clone"+i).getScanner(new - Scan())) { - assertEquals(snapshots[i-1][count++], Bytes.toString(res.getRow())); - } - Assert.assertEquals(table+"_snapshot"+i, snapshots[i-1].length, count); - } - } - } - - @Test (timeout=300000) - public void testRenameUsingSnapshots() throws Exception { - String newNS = "newNS"; - TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(newNS).build()); - for(TableName table: tables) { - int count = 0; - for(Result res: new HTable(TEST_UTIL.getConfiguration(), table).getScanner(new - Scan())) { - assertEquals(currentKeys[count++], Bytes.toString(res.getRow())); - } - TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot3", table); - final TableName newTableName = - TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3"); - TEST_UTIL.getHBaseAdmin().cloneSnapshot(table + "_snapshot3", newTableName); - Thread.sleep(1000); - count = 0; - for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new - Scan())) { - assertEquals(currentKeys[count++], Bytes.toString(res.getRow())); - } - FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath() - , LOG); - Assert.assertEquals(newTableName + "", currentKeys.length, count); - TEST_UTIL.getHBaseAdmin().flush(newTableName.toBytes()); - TEST_UTIL.getHBaseAdmin().majorCompact(newTableName.toBytes()); - TEST_UTIL.waitFor(30000, new Waiter.Predicate<IOException>() { - @Override - public boolean evaluate() throws IOException { - try { - return TEST_UTIL.getHBaseAdmin().getCompactionState(newTableName.toBytes()) == - AdminProtos.GetRegionInfoResponse.CompactionState.NONE; - } catch (InterruptedException e) { - throw new IOException(e); - } - } - }); - } - - String nextNS = "nextNS"; - TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(nextNS).build()); - for(TableName table: tables) { - TableName srcTable = TableName.valueOf(newNS + TableName.NAMESPACE_DELIM + table + "_clone3"); - TEST_UTIL.getHBaseAdmin().snapshot(table + "_snapshot4", srcTable); - TableName newTableName = - TableName.valueOf(nextNS + TableName.NAMESPACE_DELIM + table + "_clone4"); - TEST_UTIL.getHBaseAdmin().cloneSnapshot(table+"_snapshot4", newTableName); - FSUtils.logFileSystemState(TEST_UTIL.getTestFileSystem(), TEST_UTIL.getDefaultRootDirPath(), - LOG); - int count = 0; - for(Result res: new HTable(TEST_UTIL.getConfiguration(), newTableName).getScanner(new - Scan())) { - assertEquals(currentKeys[count++], Bytes.toString(res.getRow())); - } - Assert.assertEquals(newTableName + "", currentKeys.length, count); - } - } - - @Test (timeout=300000) - public void testOldDirsAreGonePostMigration() throws IOException { - FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); - Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath(); - List <String> dirs = new ArrayList<String>(NamespaceUpgrade.NON_USER_TABLE_DIRS); - // Remove those that are not renamed - dirs.remove(HConstants.HBCK_SIDELINEDIR_NAME); - dirs.remove(HConstants.SNAPSHOT_DIR_NAME); - dirs.remove(HConstants.HBASE_TEMP_DIRECTORY); - for (String dir: dirs) { - assertFalse(fs.exists(new Path(hbaseRootDir, dir))); - } - } - - @Test (timeout=300000) - public void testNewDirsArePresentPostMigration() throws IOException { - FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); - // Below list does not include 'corrupt' because there is no 'corrupt' in the tgz - String [] newdirs = new String [] {HConstants.BASE_NAMESPACE_DIR, - HConstants.HREGION_LOGDIR_NAME}; - Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath(); - for (String dir: newdirs) { - assertTrue(dir, fs.exists(new Path(hbaseRootDir, dir))); - } - } - - @Test (timeout = 300000) - public void testACLTableMigration() throws IOException { - Path rootDir = TEST_UTIL.getDataTestDirOnTestFS("testACLTable"); - FileSystem fs = TEST_UTIL.getTestFileSystem(); - Configuration conf = TEST_UTIL.getConfiguration(); - byte[] FAMILY = Bytes.toBytes("l"); - byte[] QUALIFIER = Bytes.toBytes("testUser"); - byte[] VALUE = Bytes.toBytes("RWCA"); - - // Create a Region - HTableDescriptor aclTable = new HTableDescriptor(TableName.valueOf("testACLTable")); - aclTable.addFamily(new HColumnDescriptor(FAMILY)); - FSTableDescriptors fstd = new FSTableDescriptors(fs, rootDir); - fstd.createTableDescriptor(aclTable); - HRegionInfo hriAcl = new HRegionInfo(aclTable.getTableName(), null, null); - HRegion region = HRegion.createHRegion(hriAcl, rootDir, conf, aclTable); - try { - // Create rows - Put p = new Put(Bytes.toBytes("-ROOT-")); - p.addImmutable(FAMILY, QUALIFIER, VALUE); - region.put(p); - p = new Put(Bytes.toBytes(".META.")); - p.addImmutable(FAMILY, QUALIFIER, VALUE); - region.put(p); - p = new Put(Bytes.toBytes("_acl_")); - p.addImmutable(FAMILY, QUALIFIER, VALUE); - region.put(p); - - NamespaceUpgrade upgrade = new NamespaceUpgrade(); - upgrade.updateAcls(region); - - // verify rows -ROOT- is removed - Get g = new Get(Bytes.toBytes("-ROOT-")); - Result r = region.get(g); - assertTrue(r == null || r.size() == 0); - - // verify rows _acl_ is renamed to hbase:acl - g = new Get(AccessControlLists.ACL_TABLE_NAME.toBytes()); - r = region.get(g); - assertTrue(r != null && r.size() == 1); - assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0); - - // verify rows .META. is renamed to hbase:meta - g = new Get(TableName.META_TABLE_NAME.toBytes()); - r = region.get(g); - assertTrue(r != null && r.size() == 1); - assertTrue(Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)) == 0); - } finally { - region.close(); - // Delete the region - HRegionFileSystem.deleteRegionFromFileSystem(conf, fs, - FSUtils.getTableDir(rootDir, hriAcl.getTable()), hriAcl); - } - } -}
http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java deleted file mode 100644 index ab293f2..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/migration/TestUpgradeTo96.java +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.migration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.io.FileLink; -import org.apache.hadoop.hbase.io.HFileLink; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.util.HFileV1Detector; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.hadoop.util.ToolRunner; -import org.apache.zookeeper.KeeperException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.google.protobuf.InvalidProtocolBufferException; - -/** - * Upgrade to 0.96 involves detecting HFileV1 in existing cluster, updating namespace and - * updating znodes. This class tests for HFileV1 detection and upgrading znodes. - * Uprading namespace is tested in {@link TestNamespaceUpgrade}. - */ -@Category(MediumTests.class) -public class TestUpgradeTo96 { - - static final Log LOG = LogFactory.getLog(TestUpgradeTo96.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - - /** - * underlying file system instance - */ - private static FileSystem fs; - /** - * hbase root dir - */ - private static Path hbaseRootDir; - private static ZooKeeperWatcher zkw; - /** - * replication peer znode (/hbase/replication/peers) - */ - private static String replicationPeerZnode; - /** - * znode of a table - */ - private static String tableAZnode; - private static ReplicationPeer peer1; - /** - * znode for replication peer1 (/hbase/replication/peers/1) - */ - private static String peer1Znode; - - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Start up the mini cluster on top of an 0.94 root.dir that has data from - // a 0.94 hbase run and see if we can migrate to 0.96 - TEST_UTIL.startMiniZKCluster(); - TEST_UTIL.startMiniDFSCluster(1); - - hbaseRootDir = TEST_UTIL.getDefaultRootDirPath(); - fs = FileSystem.get(TEST_UTIL.getConfiguration()); - FSUtils.setRootDir(TEST_UTIL.getConfiguration(), hbaseRootDir); - zkw = TEST_UTIL.getZooKeeperWatcher(); - - Path testdir = TEST_UTIL.getDataTestDir("TestUpgradeTo96"); - // get the untar 0.94 file structure - - set94FSLayout(testdir); - setUp94Znodes(); - } - - /** - * Lays out 0.94 file system layout using {@link TestNamespaceUpgrade} apis. - * @param testdir - * @throws IOException - * @throws Exception - */ - private static void set94FSLayout(Path testdir) throws IOException, Exception { - File untar = TestNamespaceUpgrade.untar(new File(testdir.toString())); - if (!fs.exists(hbaseRootDir.getParent())) { - // mkdir at first - fs.mkdirs(hbaseRootDir.getParent()); - } - FsShell shell = new FsShell(TEST_UTIL.getConfiguration()); - shell.run(new String[] { "-put", untar.toURI().toString(), hbaseRootDir.toString() }); - // See whats in minihdfs. - shell.run(new String[] { "-lsr", "/" }); - } - - /** - * Sets znodes used in 0.94 version. Only table and replication znodes will be upgraded to PB, - * others would be deleted. - * @throws KeeperException - */ - private static void setUp94Znodes() throws IOException, KeeperException { - // add some old znodes, which would be deleted after upgrade. - String rootRegionServerZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server"); - ZKUtil.createWithParents(zkw, rootRegionServerZnode); - ZKUtil.createWithParents(zkw, zkw.backupMasterAddressesZNode); - // add table znode, data of its children would be protobuffized - tableAZnode = ZKUtil.joinZNode(zkw.tableZNode, "a"); - ZKUtil.createWithParents(zkw, tableAZnode, - Bytes.toBytes(ZooKeeperProtos.Table.State.ENABLED.toString())); - // add replication znodes, data of its children would be protobuffized - String replicationZnode = ZKUtil.joinZNode(zkw.baseZNode, "replication"); - replicationPeerZnode = ZKUtil.joinZNode(replicationZnode, "peers"); - peer1Znode = ZKUtil.joinZNode(replicationPeerZnode, "1"); - peer1 = ReplicationPeer.newBuilder().setClusterkey("abc:123:/hbase").build(); - ZKUtil.createWithParents(zkw, peer1Znode, Bytes.toBytes(peer1.getClusterkey())); - } - - /** - * Tests a 0.94 filesystem for any HFileV1. - * @throws Exception - */ - @Test - public void testHFileV1Detector() throws Exception { - assertEquals(0, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null)); - } - - /** - * Creates a corrupt file, and run HFileV1 detector tool - * @throws Exception - */ - @Test - public void testHFileV1DetectorWithCorruptFiles() throws Exception { - // add a corrupt file. - Path tablePath = new Path(hbaseRootDir, "foo"); - FileStatus[] regionsDir = fs.listStatus(tablePath); - if (regionsDir == null) throw new IOException("No Regions found for table " + "foo"); - Path columnFamilyDir = null; - Path targetRegion = null; - for (FileStatus s : regionsDir) { - if (fs.exists(new Path(s.getPath(), HRegionFileSystem.REGION_INFO_FILE))) { - targetRegion = s.getPath(); - break; - } - } - FileStatus[] cfs = fs.listStatus(targetRegion); - for (FileStatus f : cfs) { - if (f.isDirectory()) { - columnFamilyDir = f.getPath(); - break; - } - } - LOG.debug("target columnFamilyDir: " + columnFamilyDir); - // now insert a corrupt file in the columnfamily. - Path corruptFile = new Path(columnFamilyDir, "corrupt_file"); - if (!fs.createNewFile(corruptFile)) throw new IOException("Couldn't create corrupt file: " - + corruptFile); - assertEquals(1, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null)); - // remove the corrupt file - FileSystem.get(TEST_UTIL.getConfiguration()).delete(corruptFile, false); - } - - @Test - public void testHFileLink() throws Exception { - // pass a link, and verify that correct paths are returned. - Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration()); - Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48"); - Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48"); - Path preNamespaceArchivePath = - new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48"); - Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48"); - boolean preNSTablePathExists = false; - boolean preNSArchivePathExists = false; - boolean preNSTempPathExists = false; - assertTrue(HFileLink.isHFileLink(aFileLink)); - HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink); - assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString())); - - HFileV1Detector t = new HFileV1Detector(); - t.setConf(TEST_UTIL.getConfiguration()); - FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink); - //assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp) to look. - assertTrue(fileLink.getLocations().length == 6); - for (Path p : fileLink.getLocations()) { - if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true; - if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true; - if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true; - } - assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists); - } - - @Test - public void testADirForHFileV1() throws Exception { - Path tablePath = new Path(hbaseRootDir, "foo"); - System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs)); - System.out.println("Passed: " + hbaseRootDir + "/foo"); - assertEquals(0, - ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), new String[] { "-p" - + "foo" })); - } - - @Test - public void testZnodeMigration() throws Exception { - String rootRSZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server"); - assertTrue(ZKUtil.checkExists(zkw, rootRSZnode) > -1); - ToolRunner.run(TEST_UTIL.getConfiguration(), new UpgradeTo96(), new String[] { "-execute" }); - assertEquals(-1, ZKUtil.checkExists(zkw, rootRSZnode)); - byte[] data = ZKUtil.getData(zkw, tableAZnode); - assertTrue(ProtobufUtil.isPBMagicPrefix(data)); - checkTableState(data, ZooKeeperProtos.Table.State.ENABLED); - // ensure replication znodes are there, and protobuffed. - data = ZKUtil.getData(zkw, peer1Znode); - assertTrue(ProtobufUtil.isPBMagicPrefix(data)); - checkReplicationPeerData(data, peer1); - } - - private void checkTableState(byte[] data, State expectedState) - throws InvalidProtocolBufferException { - ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder(); - int magicLen = ProtobufUtil.lengthOfPBMagic(); - ZooKeeperProtos.Table t = builder.mergeFrom(data, magicLen, data.length - magicLen).build(); - assertTrue(t.getState() == expectedState); - } - - private void checkReplicationPeerData(byte[] data, ReplicationPeer peer) - throws InvalidProtocolBufferException { - int magicLen = ProtobufUtil.lengthOfPBMagic(); - ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder(); - assertEquals(builder.mergeFrom(data, magicLen, data.length - magicLen).build().getClusterkey(), - peer.getClusterkey()); - - } - - @AfterClass - public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniHBaseCluster(); - TEST_UTIL.shutdownMiniDFSCluster(); - TEST_UTIL.shutdownMiniZKCluster(); - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java index 8de605d..6cec1e2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestEndToEndSplitTransaction.java @@ -34,16 +34,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Chore; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.LargeTests; import org.apache.hadoop.hbase.NotServingRegionException; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; @@ -52,14 +51,13 @@ import org.apache.hadoop.hbase.client.MetaScanner; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; +import org.apache.hadoop.hbase.ipc.PayloadCarryingRpcController; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.StoppableImplementation; @@ -74,6 +72,7 @@ import com.google.common.collect.Sets; import com.google.protobuf.ServiceException; @Category(LargeTests.class) +@SuppressWarnings("deprecation") public class TestEndToEndSplitTransaction { private static final Log LOG = LogFactory.getLog(TestEndToEndSplitTransaction.class); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); @@ -109,7 +108,6 @@ public class TestEndToEndSplitTransaction { .getRegionName(); HRegion region = server.getRegion(regionName); SplitTransaction split = new SplitTransaction(region, splitRow); - split.useZKForAssignment = ConfigUtil.useZKForAssignment(conf); split.prepare(); // 1. phase I @@ -126,14 +124,10 @@ public class TestEndToEndSplitTransaction { // 3. finish phase II // note that this replicates some code from SplitTransaction // 2nd daughter first - if (split.useZKForAssignment) { - server.postOpenDeployTasks(regions.getSecond()); - } else { server.reportRegionStateTransition( RegionServerStatusProtos.RegionStateTransition.TransitionCode.SPLIT, region.getRegionInfo(), regions.getFirst().getRegionInfo(), regions.getSecond().getRegionInfo()); - } // Add to online regions server.addToOnlineRegions(regions.getSecond()); @@ -143,21 +137,11 @@ public class TestEndToEndSplitTransaction { // past splitkey is ok. assertTrue(test(con, tableName, lastRow, server)); - // first daughter second - if (split.useZKForAssignment) { - server.postOpenDeployTasks(regions.getFirst()); - } // Add to online regions server.addToOnlineRegions(regions.getFirst()); assertTrue(test(con, tableName, firstRow, server)); assertTrue(test(con, tableName, lastRow, server)); - if (split.useZKForAssignment) { - // 4. phase III - ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) - .getSplitTransactionCoordination().completeSplitTransaction(server, regions.getFirst(), - regions.getSecond(), split.std, region); - } assertTrue(test(con, tableName, firstRow, server)); assertTrue(test(con, tableName, lastRow, server)); } @@ -341,7 +325,7 @@ public class TestEndToEndSplitTransaction { verifyStartEndKeys(keys); //HTable.getRegionsInfo() - Map<HRegionInfo, ServerName> regions = table.getRegionLocations(); + Map<HRegionInfo, ServerName> regions = table.getRegionLocations(); verifyTableRegions(regions.keySet()); } finally { IOUtils.closeQuietly(table); http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index d9883e0..ac5e41e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -37,13 +37,12 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionReplicaUtil; @@ -94,7 +93,8 @@ public class TestRegionMergeTransactionOnCluster { private static HMaster master; private static Admin admin; - static void setupOnce() throws Exception { + @BeforeClass + public static void beforeAllTests() throws Exception { // Start a cluster TEST_UTIL.startMiniCluster(NB_SERVERS); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); @@ -103,13 +103,6 @@ public class TestRegionMergeTransactionOnCluster { admin = TEST_UTIL.getHBaseAdmin(); } - @BeforeClass - public static void beforeAllTests() throws Exception { - // Use ZK for region assignment - TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); - setupOnce(); - } - @AfterClass public static void afterAllTests() throws Exception { TEST_UTIL.shutdownMiniCluster(); @@ -148,7 +141,7 @@ public class TestRegionMergeTransactionOnCluster { } // We should not be able to assign it again - am.assign(hri, true, true); + am.assign(hri, true); assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri)); assertTrue(regionStates.isRegionInState(hri, State.MERGED)); @@ -162,6 +155,7 @@ public class TestRegionMergeTransactionOnCluster { table.close(); } + @SuppressWarnings("deprecation") @Test public void testCleanMergeReference() throws Exception { LOG.info("Starting testCleanMergeReference"); http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java index 9f64a7c..a7067f1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java @@ -45,9 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.util.StringUtils; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -94,26 +92,18 @@ public class TestRegionReplicas { @AfterClass public static void afterClass() throws Exception { + HRegionServer.TEST_SKIP_REPORTING_TRANSITION = false; table.close(); HTU.shutdownMiniCluster(); } - @After - public void after() throws Exception { - // Clean the state if the test failed before cleaning the znode - // It does not manage all bad failures, so if there are multiple failures, only - // the first one should be looked at. - ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hriPrimary); - } - private HRegionServer getRS() { return HTU.getMiniHBaseCluster().getRegionServer(0); } private void openRegion(HRegionInfo hri) throws Exception { - ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); // first version is '0' - AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null); + AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, null, null); AdminProtos.OpenRegionResponse responseOpen = getRS().getRSRpcServices().openRegion(null, orr); Assert.assertTrue(responseOpen.getOpeningStateCount() == 1); Assert.assertTrue(responseOpen.getOpeningState(0). @@ -122,16 +112,12 @@ public class TestRegionReplicas { } private void closeRegion(HRegionInfo hri) throws Exception { - ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); - AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(getRS().getServerName(), - hri.getEncodedName(), true); + hri.getEncodedName()); AdminProtos.CloseRegionResponse responseClose = getRS().getRSRpcServices().closeRegion(null, crr); Assert.assertTrue(responseClose.getClosed()); checkRegionIsClosed(hri.getEncodedName()); - - ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), getRS().getServerName()); } private void checkRegionIsOpened(String encodedRegionName) throws Exception { @@ -141,9 +127,6 @@ public class TestRegionReplicas { } Assert.assertTrue(getRS().getRegionByEncodedName(encodedRegionName).isAvailable()); - - Assert.assertTrue( - ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), encodedRegionName, getRS().getServerName())); } http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java index 0843abe..3164f1d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java @@ -26,29 +26,22 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; -import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; -import org.apache.hadoop.hbase.coordination.ZkOpenRegionCoordination; -import org.apache.hadoop.hbase.executor.EventType; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionRequest; import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.NodeExistsException; -import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -77,7 +70,6 @@ public class TestRegionServerNoMaster { @BeforeClass public static void before() throws Exception { - HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true); HTU.startMiniCluster(NB_SERVERS); final byte[] tableName = Bytes.toBytes(TestRegionServerNoMaster.class.getSimpleName()); @@ -94,27 +86,37 @@ public class TestRegionServerNoMaster { } public static void stopMasterAndAssignMeta(HBaseTestingUtility HTU) - throws NodeExistsException, KeeperException, IOException, InterruptedException { - // No master - HTU.getHBaseCluster().getMaster().stopMaster(); + throws IOException, InterruptedException { + // Stop master + HMaster master = HTU.getHBaseCluster().getMaster(); + ServerName masterAddr = master.getServerName(); + master.stopMaster(); Log.info("Waiting until master thread exits"); while (HTU.getHBaseCluster().getMasterThread() != null && HTU.getHBaseCluster().getMasterThread().isAlive()) { Threads.sleep(100); } + + HRegionServer.TEST_SKIP_REPORTING_TRANSITION = true; // Master is down, so is the meta. We need to assign it somewhere // so that regions can be assigned during the mocking phase. - HRegionServer hrs = HTU.getHBaseCluster().getRegionServer(0); + HRegionServer hrs = HTU.getHBaseCluster() + .getLiveRegionServerThreads().get(0).getRegionServer(); ZooKeeperWatcher zkw = hrs.getZooKeeper(); - ZKAssign.createNodeOffline( - zkw, HRegionInfo.FIRST_META_REGIONINFO, hrs.getServerName()); + MetaTableLocator mtl = new MetaTableLocator(); + ServerName sn = mtl.getMetaRegionLocation(zkw); + if (sn != null && !masterAddr.equals(sn)) { + return; + } + ProtobufUtil.openRegion(hrs.getRSRpcServices(), hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO); - MetaTableLocator mtl = new MetaTableLocator(); while (true) { - ServerName sn = mtl.getMetaRegionLocation(zkw); - if (sn != null && sn.equals(hrs.getServerName())) { + sn = mtl.getMetaRegionLocation(zkw); + if (sn != null && sn.equals(hrs.getServerName()) + && hrs.onlineRegions.containsKey( + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) { break; } Thread.sleep(100); @@ -135,19 +137,11 @@ public class TestRegionServerNoMaster { @AfterClass public static void afterClass() throws Exception { + HRegionServer.TEST_SKIP_REPORTING_TRANSITION = false; table.close(); HTU.shutdownMiniCluster(); } - @After - public void after() throws Exception { - // Clean the state if the test failed before cleaning the znode - // It does not manage all bad failures, so if there are multiple failures, only - // the first one should be looked at. - ZKAssign.deleteNodeFailSilent(HTU.getZooKeeperWatcher(), hri); - } - - private static HRegionServer getRS() { return HTU.getHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer(); } @@ -157,11 +151,8 @@ public class TestRegionServerNoMaster { * Reopen the region. Reused in multiple tests as we always leave the region open after a test. */ private void reopenRegion() throws Exception { - // We reopen. We need a ZK node here, as a open is always triggered by a master. - ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); - // first version is '0' AdminProtos.OpenRegionRequest orr = - RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, 0, null, null); + RequestConverter.buildOpenRegionRequest(getRS().getServerName(), hri, null, null); AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr); Assert.assertTrue(responseOpen.getOpeningStateCount() == 1); Assert.assertTrue(responseOpen.getOpeningState(0). @@ -178,10 +169,6 @@ public class TestRegionServerNoMaster { } Assert.assertTrue(getRS().getRegion(regionName).isAvailable()); - - Assert.assertTrue( - ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), - getRS().getServerName())); } @@ -196,8 +183,6 @@ public class TestRegionServerNoMaster { } catch (NotServingRegionException expected) { // That's how it work: if the region is closed we have an exception. } - - // We don't delete the znode here, because there is not always a znode. } @@ -207,7 +192,7 @@ public class TestRegionServerNoMaster { private void closeNoZK() throws Exception { // no transition in ZK AdminProtos.CloseRegionRequest crr = - RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); + RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName); AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(responseClose.getClosed()); @@ -222,42 +207,6 @@ public class TestRegionServerNoMaster { reopenRegion(); } - @Test(timeout = 60000) - public void testCloseByMasterWithoutZNode() throws Exception { - - // Transition in ZK on. This should fail, as there is no znode - AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest( - getRS().getServerName(), regionName, true); - AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr); - Assert.assertTrue(responseClose.getClosed()); - - // now waiting. After a while, the transition should be done - while (!getRS().getRegionsInTransitionInRS().isEmpty()) { - Thread.sleep(1); - } - - // the region is still available, the close got rejected at the end - Assert.assertTrue("The close should have failed", getRS().getRegion(regionName).isAvailable()); - } - - @Test(timeout = 60000) - public void testOpenCloseByMasterWithZNode() throws Exception { - - ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); - - AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest( - getRS().getServerName(), regionName, true); - AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr); - Assert.assertTrue(responseClose.getClosed()); - - checkRegionIsClosed(); - - ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), - getRS().getServerName()); - - reopenRegion(); - } - /** * Test that we can send multiple openRegion to the region server. * This is used when: @@ -275,13 +224,10 @@ public class TestRegionServerNoMaster { closeNoZK(); checkRegionIsClosed(); - // We reopen. We need a ZK node here, as a open is always triggered by a master. - ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); - // We're sending multiple requests in a row. The region server must handle this nicely. for (int i = 0; i < 10; i++) { AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest( - getRS().getServerName(), hri, 0, null, null); + getRS().getServerName(), hri, null, null); AdminProtos.OpenRegionResponse responseOpen = getRS().rpcServices.openRegion(null, orr); Assert.assertTrue(responseOpen.getOpeningStateCount() == 1); @@ -307,7 +253,7 @@ public class TestRegionServerNoMaster { // fake region to be closing now, need to clear state afterwards getRS().regionsInTransitionInRS.put(hri.getEncodedNameAsBytes(), Boolean.FALSE); AdminProtos.OpenRegionRequest orr = - RequestConverter.buildOpenRegionRequest(sn, hri, 0, null, null); + RequestConverter.buildOpenRegionRequest(sn, hri, null, null); getRS().rpcServices.openRegion(null, orr); Assert.fail("The closing region should not be opened"); } catch (ServiceException se) { @@ -320,12 +266,9 @@ public class TestRegionServerNoMaster { @Test(timeout = 60000) public void testMultipleCloseFromMaster() throws Exception { - - // As opening, we must support multiple requests on the same region - ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); for (int i = 0; i < 10; i++) { AdminProtos.CloseRegionRequest crr = - RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, 0, null, true); + RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, null); try { AdminProtos.CloseRegionResponse responseClose = getRS().rpcServices.closeRegion(null, crr); Assert.assertEquals("The first request should succeeds", 0, i); @@ -338,11 +281,6 @@ public class TestRegionServerNoMaster { checkRegionIsClosed(); - Assert.assertTrue( - ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), - getRS().getServerName()) - ); - reopenRegion(); } @@ -356,12 +294,11 @@ public class TestRegionServerNoMaster { checkRegionIsClosed(); // Let do the initial steps, without having a handler - ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); // That's a close without ZK. AdminProtos.CloseRegionRequest crr = - RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); + RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName); try { getRS().rpcServices.closeRegion(null, crr); Assert.assertTrue(false); @@ -375,89 +312,11 @@ public class TestRegionServerNoMaster { // Let's start the open handler HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); - BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); - csm.initialize(getRS()); - csm.start(); - - ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = - new ZkOpenRegionCoordination.ZkOpenRegionDetails(); - zkCrd.setServerName(getRS().getServerName()); - zkCrd.setVersionOfOfflineNode(0); - - getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, - csm.getOpenRegionCoordination(), zkCrd)); + getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd)); // The open handler should have removed the region from RIT but kept the region closed checkRegionIsClosed(); - // The open handler should have updated the value in ZK. - Assert.assertTrue(ZKAssign.deleteNode( - getRS().getZooKeeper(), hri.getEncodedName(), - EventType.RS_ZK_REGION_FAILED_OPEN, 1) - ); - - reopenRegion(); - } - - /** - * Test an open then a close with ZK. This is going to mess-up the ZK states, so - * the opening will fail as well because it doesn't find what it expects in ZK. - */ - @Test(timeout = 60000) - public void testCancelOpeningWithZK() throws Exception { - // We close - closeNoZK(); - checkRegionIsClosed(); - - // Let do the initial steps, without having a handler - getRS().getRegionsInTransitionInRS().put(hri.getEncodedNameAsBytes(), Boolean.TRUE); - - // That's a close without ZK. - ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, getRS().getServerName()); - AdminProtos.CloseRegionRequest crr = - RequestConverter.buildCloseRegionRequest(getRS().getServerName(), regionName, false); - try { - getRS().rpcServices.closeRegion(null, crr); - Assert.assertTrue(false); - } catch (ServiceException expected) { - Assert.assertTrue(expected.getCause() instanceof NotServingRegionException); - } - - // The close should have left the ZK state as it is: it's the job the AM to delete it - Assert.assertTrue(ZKAssign.deleteNode( - getRS().getZooKeeper(), hri.getEncodedName(), - EventType.M_ZK_REGION_CLOSING, 0) - ); - - // The state in RIT should have changed to close - Assert.assertEquals(Boolean.FALSE, getRS().getRegionsInTransitionInRS().get( - hri.getEncodedNameAsBytes())); - - // Let's start the open handler - // It should not succeed for two reasons: - // 1) There is no ZK node - // 2) The region in RIT was changed. - // The order is more or less implementation dependant. - HTableDescriptor htd = getRS().tableDescriptors.get(hri.getTable()); - - BaseCoordinatedStateManager csm = new ZkCoordinatedStateManager(); - csm.initialize(getRS()); - csm.start(); - - ZkOpenRegionCoordination.ZkOpenRegionDetails zkCrd = - new ZkOpenRegionCoordination.ZkOpenRegionDetails(); - zkCrd.setServerName(getRS().getServerName()); - zkCrd.setVersionOfOfflineNode(0); - - getRS().service.submit(new OpenRegionHandler(getRS(), getRS(), hri, htd, - csm.getOpenRegionCoordination(), zkCrd)); - - // The open handler should have removed the region from RIT but kept the region closed - checkRegionIsClosed(); - - // We should not find any znode here. - Assert.assertEquals(-1, ZKAssign.getVersion(HTU.getZooKeeperWatcher(), hri)); - reopenRegion(); } @@ -473,7 +332,7 @@ public class TestRegionServerNoMaster { ServerName earlierServerName = ServerName.valueOf(sn.getHostname(), sn.getPort(), 1); try { - CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName, true); + CloseRegionRequest request = RequestConverter.buildCloseRegionRequest(earlierServerName, regionName); getRS().getRSRpcServices().closeRegion(null, request); Assert.fail("The closeRegion should have been rejected"); } catch (ServiceException se) { @@ -485,7 +344,7 @@ public class TestRegionServerNoMaster { closeNoZK(); try { AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest( - earlierServerName, hri, 0, null, null); + earlierServerName, hri, null, null); getRS().getRSRpcServices().openRegion(null, orr); Assert.fail("The openRegion should have been rejected"); } catch (ServiceException se) { http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index da4b3d9..3818b60 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -48,15 +47,14 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.LargeTests; import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Consistency; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -68,14 +66,10 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TestReplicasClient.SlowMeCopro; -import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination; -import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination; import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.RegionState; @@ -90,12 +84,9 @@ import org.apache.hadoop.hbase.util.HBaseFsck; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; -import org.apache.zookeeper.data.Stat; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -120,26 +111,15 @@ public class TestSplitTransactionOnCluster { private static final int NB_SERVERS = 3; private static CountDownLatch latch = new CountDownLatch(1); private static volatile boolean secondSplit = false; - private static volatile boolean callRollBack = false; - private static volatile boolean firstSplitCompleted = false; - private static boolean useZKForAssignment = true; static final HBaseTestingUtility TESTING_UTIL = new HBaseTestingUtility(); - static void setupOnce() throws Exception { + @BeforeClass public static void before() throws Exception { TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000); - useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean( - "hbase.assignment.usezk", false); TESTING_UTIL.startMiniCluster(NB_SERVERS); } - @BeforeClass public static void before() throws Exception { - // Use ZK for region assignment - TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); - setupOnce(); - } - @AfterClass public static void after() throws Exception { TESTING_UTIL.shutdownMiniCluster(); } @@ -181,100 +161,6 @@ public class TestSplitTransactionOnCluster { return hri; } - @SuppressWarnings("deprecation") - @Test(timeout = 60000) - public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception { - final TableName tableName = - TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack"); - - if (!useZKForAssignment) { - // This test doesn't apply if not using ZK for assignment - return; - } - - try { - // Create table then get the single region for our new table. - HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf")); - final List<HRegion> regions = cluster.getRegions(tableName); - HRegionInfo hri = getAndCheckSingleTableRegion(regions); - int regionServerIndex = cluster.getServerWith(regions.get(0).getRegionName()); - final HRegionServer regionServer = cluster.getRegionServer(regionServerIndex); - insertData(tableName.getName(), admin, t); - t.close(); - - // Turn off balancer so it doesn't cut in and mess up our placements. - this.admin.setBalancerRunning(false, true); - // Turn off the meta scanner so it don't remove parent on us. - cluster.getMaster().setCatalogJanitorEnabled(false); - - // find a splittable region - final HRegion region = findSplittableRegion(regions); - assertTrue("not able to find a splittable region", region != null); - MockedCoordinatedStateManager cp = new MockedCoordinatedStateManager(); - cp.initialize(regionServer, region); - cp.start(); - regionServer.csm = cp; - - new Thread() { - @Override - public void run() { - SplitTransaction st = null; - st = new MockedSplitTransaction(region, Bytes.toBytes("row2")); - try { - st.prepare(); - st.execute(regionServer, regionServer); - } catch (IOException e) { - - } - } - }.start(); - for (int i = 0; !callRollBack && i < 100; i++) { - Thread.sleep(100); - } - assertTrue("Waited too long for rollback", callRollBack); - SplitTransaction st = new MockedSplitTransaction(region, Bytes.toBytes("row3")); - try { - secondSplit = true; - // make region splittable - region.initialize(); - st.prepare(); - st.execute(regionServer, regionServer); - } catch (IOException e) { - LOG.debug("Rollback started :"+ e.getMessage()); - st.rollback(regionServer, regionServer); - } - for (int i=0; !firstSplitCompleted && i<100; i++) { - Thread.sleep(100); - } - assertTrue("fist split did not complete", firstSplitCompleted); - - RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates(); - Map<String, RegionState> rit = regionStates.getRegionsInTransition(); - - for (int i=0; rit.containsKey(hri.getTable()) && i<100; i++) { - Thread.sleep(100); - } - assertFalse("region still in transition", rit.containsKey( - rit.containsKey(hri.getTable()))); - - List<HRegion> onlineRegions = regionServer.getOnlineRegions(tableName); - // Region server side split is successful. - assertEquals("The parent region should be splitted", 2, onlineRegions.size()); - //Should be present in RIT - List<HRegionInfo> regionsOfTable = cluster.getMaster().getAssignmentManager() - .getRegionStates().getRegionsOfTable(tableName); - // Master side should also reflect the same - assertEquals("No of regions in master", 2, regionsOfTable.size()); - } finally { - admin.setBalancerRunning(true, false); - secondSplit = false; - firstSplitCompleted = false; - callRollBack = false; - cluster.getMaster().setCatalogJanitorEnabled(true); - TESTING_UTIL.deleteTable(tableName); - } - } - @Test(timeout = 60000) public void testRITStateForRollback() throws Exception { final TableName tableName = @@ -382,20 +268,15 @@ public class TestSplitTransactionOnCluster { /** * A test that intentionally has master fail the processing of the split message. - * Tests that the regionserver split ephemeral node gets cleaned up if it - * crashes and that after we process server shutdown, the daughters are up on - * line. + * Tests that after we process server shutdown, the daughters are up on line. * @throws IOException * @throws InterruptedException - * @throws NodeExistsException - * @throws KeeperException - * @throws DeserializationException + * @throws ServiceException */ - @Test (timeout = 300000) public void testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling() - throws IOException, InterruptedException, NodeExistsException, KeeperException, - DeserializationException, ServiceException { + @Test (timeout = 300000) public void testRSSplitDaughtersAreOnlinedAfterShutdownHandling() + throws IOException, InterruptedException, ServiceException { final byte [] tableName = - Bytes.toBytes("testRSSplitEphemeralsDisappearButDaughtersAreOnlinedAfterShutdownHandling"); + Bytes.toBytes("testRSSplitDaughtersAreOnlinedAfterShutdownHandling"); // Create table then get the single region for our new table. HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); @@ -418,48 +299,15 @@ public class TestSplitTransactionOnCluster { // Now, before we split, set special flag in master, a flag that has // it FAIL the processing of split. AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true; - // Now try splitting and it should work. - split(hri, server, regionCount); - - String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(), - hri.getEncodedName()); - RegionTransition rt = null; - Stat stats = null; - List<HRegion> daughters = null; - if (useZKForAssignment) { - daughters = checkAndGetDaughters(tableName); - - // Wait till the znode moved to SPLIT - for (int i=0; i<100; i++) { - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(), - hri.getEncodedName())); - if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break; - Thread.sleep(100); - } - LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)); - // Now crash the server, for ZK-less assignment, the server is auto aborted - cluster.abortRegionServer(tableRegionIndex); + try { + // Now try splitting and it should work. + split(hri, server, regionCount); + } catch (RegionServerStoppedException rsse) { + // Expected. The regionserver should crash } + waitUntilRegionServerDead(); awaitDaughters(tableName, 2); - if (useZKForAssignment) { - regions = cluster.getRegions(tableName); - for (HRegion r: regions) { - assertTrue(daughters.contains(r)); - } - - // Finally assert that the ephemeral SPLIT znode was cleaned up. - for (int i=0; i<100; i++) { - // wait a bit (10s max) for the node to disappear - stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path, false); - if (stats == null) break; - Thread.sleep(100); - } - LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats); - assertTrue(stats == null); - } } finally { // Set this flag back. AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false; @@ -495,15 +343,8 @@ public class TestSplitTransactionOnCluster { HRegionServer server = cluster.getRegionServer(tableRegionIndex); printOutRegions(server, "Initial regions: "); int regionCount = ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size(); - // Insert into zk a blocking znode, a znode of same name as region - // so it gets in way of our splitting. - ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1); - if (useZKForAssignment) { - ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(), - hri, fakedServer); - } else { - regionStates.updateRegionState(hri, RegionState.State.CLOSING); - } + regionStates.updateRegionState(hri, RegionState.State.CLOSING); + // Now try splitting.... should fail. And each should successfully // rollback. this.admin.split(hri.getRegionNameAsString()); @@ -515,13 +356,8 @@ public class TestSplitTransactionOnCluster { assertEquals(regionCount, ProtobufUtil.getOnlineRegions( server.getRSRpcServices()).size()); } - if (useZKForAssignment) { - // Now clear the zknode - ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(), - hri, fakedServer); - } else { - regionStates.regionOnline(hri, server.getServerName()); - } + regionStates.regionOnline(hri, server.getServerName()); + // Now try splitting and it should work. split(hri, server, regionCount); // Get daughters @@ -541,7 +377,7 @@ public class TestSplitTransactionOnCluster { * @throws InterruptedException */ @Test (timeout=300000) public void testShutdownFixupWhenDaughterHasSplit() - throws IOException, InterruptedException, ServiceException { + throws IOException, InterruptedException { final byte [] tableName = Bytes.toBytes("testShutdownFixupWhenDaughterHasSplit"); @@ -698,103 +534,6 @@ public class TestSplitTransactionOnCluster { } /** - * Verifies HBASE-5806. When splitting is partially done and the master goes down - * when the SPLIT node is in either SPLIT or SPLITTING state. - * - * @throws IOException - * @throws InterruptedException - * @throws NodeExistsException - * @throws KeeperException - * @throws DeserializationException - */ - @Test(timeout = 400000) - public void testMasterRestartWhenSplittingIsPartial() - throws IOException, InterruptedException, NodeExistsException, - KeeperException, DeserializationException, ServiceException { - final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial"); - - if (!useZKForAssignment) { - // This test doesn't apply if not using ZK for assignment - return; - } - - // Create table then get the single region for our new table. - HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY); - List<HRegion> regions = cluster.getRegions(tableName); - HRegionInfo hri = getAndCheckSingleTableRegion(regions); - - int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri); - - // Turn off balancer so it doesn't cut in and mess up our placements. - this.admin.setBalancerRunning(false, true); - // Turn off the meta scanner so it don't remove parent on us. - cluster.getMaster().setCatalogJanitorEnabled(false); - ZooKeeperWatcher zkw = new ZooKeeperWatcher(t.getConfiguration(), - "testMasterRestartWhenSplittingIsPartial", new UselessTestAbortable()); - try { - // Add a bit of load up into the table so splittable. - TESTING_UTIL.loadTable(t, HConstants.CATALOG_FAMILY, false); - // Get region pre-split. - HRegionServer server = cluster.getRegionServer(tableRegionIndex); - printOutRegions(server, "Initial regions: "); - // Now, before we split, set special flag in master, a flag that has - // it FAIL the processing of split. - AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true; - // Now try splitting and it should work. - - this.admin.split(hri.getRegionNameAsString()); - checkAndGetDaughters(tableName); - // Assert the ephemeral node is up in zk. - String path = ZKAssign.getNodeName(zkw, hri.getEncodedName()); - Stat stats = zkw.getRecoverableZooKeeper().exists(path, false); - LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" - + stats); - byte[] bytes = ZKAssign.getData(zkw, hri.getEncodedName()); - RegionTransition rtd = RegionTransition.parseFrom(bytes); - // State could be SPLIT or SPLITTING. - assertTrue(rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLIT) - || rtd.getEventType().equals(EventType.RS_ZK_REGION_SPLITTING)); - - // abort and wait for new master. - MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster(); - - this.admin = new HBaseAdmin(TESTING_UTIL.getConfiguration()); - - // Update the region to be offline and split, so that HRegionInfo#equals - // returns true in checking rebuilt region states map. - hri.setOffline(true); - hri.setSplit(true); - ServerName regionServerOfRegion = master.getAssignmentManager() - .getRegionStates().getRegionServerOfRegion(hri); - assertTrue(regionServerOfRegion != null); - - // Remove the block so that split can move ahead. - AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false; - String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); - Stat stat = new Stat(); - byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat); - // ZKUtil.create - for (int i=0; data != null && i<60; i++) { - Thread.sleep(1000); - data = ZKUtil.getDataNoWatch(zkw, node, stat); - } - assertNull("Waited too long for ZK node to be removed: "+node, data); - RegionStates regionStates = master.getAssignmentManager().getRegionStates(); - assertTrue("Split parent should be in SPLIT state", - regionStates.isRegionInState(hri, State.SPLIT)); - regionServerOfRegion = regionStates.getRegionServerOfRegion(hri); - assertTrue(regionServerOfRegion == null); - } finally { - // Set this flag back. - AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false; - admin.setBalancerRunning(true, false); - cluster.getMaster().setCatalogJanitorEnabled(true); - t.close(); - zkw.close(); - } - } - - /** * Verifies HBASE-5806. Here the case is that splitting is completed but before the * CJ could remove the parent region the master is killed and restarted. * @throws IOException @@ -830,20 +569,6 @@ public class TestSplitTransactionOnCluster { this.admin.split(hri.getRegionNameAsString()); checkAndGetDaughters(tableName); - // Assert the ephemeral node is up in zk. - String path = ZKAssign.getNodeName(zkw, hri.getEncodedName()); - Stat stats = zkw.getRecoverableZooKeeper().exists(path, false); - LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" - + stats); - String node = ZKAssign.getNodeName(zkw, hri.getEncodedName()); - Stat stat = new Stat(); - byte[] data = ZKUtil.getDataNoWatch(zkw, node, stat); - // ZKUtil.create - for (int i=0; data != null && i<60; i++) { - Thread.sleep(1000); - data = ZKUtil.getDataNoWatch(zkw, node, stat); - } - assertNull("Waited too long for ZK node to be removed: "+node, data); MockMasterWithoutCatalogJanitor master = abortAndWaitForMaster(); @@ -885,7 +610,6 @@ public class TestSplitTransactionOnCluster { @Test(timeout = 60000) public void testTableExistsIfTheSpecifiedTableRegionIsSplitParent() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL); final TableName tableName = TableName.valueOf("testTableExistsIfTheSpecifiedTableRegionIsSplitParent"); // Create table then get the single region for our new table. @@ -916,11 +640,6 @@ public class TestSplitTransactionOnCluster { tableName); assertEquals("The specified table should present.", true, tableExists); } finally { - if (regions != null) { - String node = ZKAssign.getNodeName(zkw, regions.get(0).getRegionInfo() - .getEncodedName()); - ZKUtil.deleteNodeFailSilent(zkw, node); - } admin.setBalancerRunning(true, false); cluster.getMaster().setCatalogJanitorEnabled(true); t.close(); @@ -929,7 +648,6 @@ public class TestSplitTransactionOnCluster { @Test public void testSplitWithRegionReplicas() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TESTING_UTIL); final TableName tableName = TableName.valueOf("foobar"); HTableDescriptor htd = TESTING_UTIL.createTableDescriptor("foobar"); @@ -938,7 +656,6 @@ public class TestSplitTransactionOnCluster { // Create table then get the single region for our new table. HTable t = TESTING_UTIL.createTable(htd, new byte[][]{Bytes.toBytes("cf")}, TESTING_UTIL.getConfiguration()); - int count; List<HRegion> oldRegions; do { oldRegions = cluster.getRegions(tableName); @@ -960,9 +677,6 @@ public class TestSplitTransactionOnCluster { regionServerIndex = cluster.getServerWith(region.getRegionName()); regionServer = cluster.getRegionServer(regionServerIndex); assertTrue("not able to find a splittable region", region != null); - String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), - region.getRegionInfo().getEncodedName()); - regionServer.getZooKeeper().sync(node); SplitTransaction st = new SplitTransaction(region, Bytes.toBytes("row2")); try { st.prepare(); @@ -1093,7 +807,7 @@ public class TestSplitTransactionOnCluster { } // We should not be able to assign it again - am.assign(hri, true, true); + am.assign(hri, true); assertFalse("Split region can't be assigned", regionStates.isRegionInTransition(hri)); assertTrue(regionStates.isRegionInState(hri, State.SPLIT)); @@ -1176,17 +890,6 @@ public class TestSplitTransactionOnCluster { throw new SplittingNodeCreationFailedException (); } }; - String node = ZKAssign.getNodeName(regionServer.getZooKeeper(), - region.getRegionInfo().getEncodedName()); - regionServer.getZooKeeper().sync(node); - for (int i = 0; i < 100; i++) { - // We expect the znode to be deleted by this time. Here the - // znode could be in OPENED state and the - // master has not yet deleted the znode. - if (ZKUtil.checkExists(regionServer.getZooKeeper(), node) != -1) { - Thread.sleep(100); - } - } try { st.prepare(); st.execute(regionServer, regionServer); @@ -1196,13 +899,7 @@ public class TestSplitTransactionOnCluster { // This will at least make the test to fail; assertTrue("Should be instance of CreateSplittingNodeFailedException", e instanceof SplittingNodeCreationFailedException ); - node = ZKAssign.getNodeName(regionServer.getZooKeeper(), - region.getRegionInfo().getEncodedName()); - { - assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); - } assertTrue(st.rollback(regionServer, regionServer)); - assertTrue(ZKUtil.checkExists(regionServer.getZooKeeper(), node) == -1); } } finally { TESTING_UTIL.deleteTable(tableName); @@ -1213,8 +910,6 @@ public class TestSplitTransactionOnCluster { public void initialize(Server server, HRegion region) { this.server = server; this.watcher = server.getZooKeeper(); - splitTransactionCoordination = new MockedSplitTransactionCoordination(this, watcher, region); - closeRegionCoordination = new ZkCloseRegionCoordination(this, watcher); } } @@ -1237,41 +932,7 @@ public class TestSplitTransactionOnCluster { } return super.rollback(server, services); } - - - } - - public static class MockedSplitTransactionCoordination extends ZKSplitTransactionCoordination { - - private HRegion currentRegion; - - public MockedSplitTransactionCoordination(CoordinatedStateManager coordinationProvider, - ZooKeeperWatcher watcher, HRegion region) { - super(coordinationProvider, watcher); - currentRegion = region; - } - - @Override - public void completeSplitTransaction(RegionServerServices services, HRegion a, HRegion b, - SplitTransactionDetails std, HRegion parent) throws IOException { - if (this.currentRegion.getRegionInfo().getTable().getNameAsString() - .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { - try { - if (!secondSplit){ - callRollBack = true; - latch.await(); - } - } catch (InterruptedException e) { - } - } - super.completeSplitTransaction(services, a, b, std, parent); - if (this.currentRegion.getRegionInfo().getTable().getNameAsString() - .equals("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack")) { - firstSplitCompleted = true; - } - } - } private HRegion findSplittableRegion(final List<HRegion> regions) throws InterruptedException { for (int i = 0; i < 5; ++i) { @@ -1313,22 +974,14 @@ public class TestSplitTransactionOnCluster { private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount) throws IOException, InterruptedException { this.admin.split(hri.getRegionNameAsString()); - try { - for (int i = 0; ProtobufUtil.getOnlineRegions( - server.getRSRpcServices()).size() <= regionCount && i < 300; i++) { - LOG.debug("Waiting on region to split"); - Thread.sleep(100); - } - - assertFalse("Waited too long for split", - ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount); - } catch (RegionServerStoppedException e) { - if (useZKForAssignment) { - // If not using ZK for assignment, the exception may be expected. - LOG.error(e); - throw e; - } + for (int i = 0; ProtobufUtil.getOnlineRegions( + server.getRSRpcServices()).size() <= regionCount && i < 300; i++) { + LOG.debug("Waiting on region to split"); + Thread.sleep(100); } + + assertFalse("Waited too long for split", + ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount); } /** @@ -1411,13 +1064,14 @@ public class TestSplitTransactionOnCluster { private void waitUntilRegionServerDead() throws InterruptedException, InterruptedIOException { // Wait until the master processes the RS shutdown - for (int i=0; cluster.getMaster().getClusterStatus(). - getServers().size() > NB_SERVERS && i<100; i++) { + for (int i=0; (cluster.getMaster().getClusterStatus().getServers().size() > NB_SERVERS + || cluster.getLiveRegionServerThreads().size() > NB_SERVERS) && i<100; i++) { LOG.info("Waiting on server to go down"); Thread.sleep(100); } - assertFalse("Waited too long for RS to die", cluster.getMaster().getClusterStatus(). - getServers().size() > NB_SERVERS); + assertFalse("Waited too long for RS to die", + cluster.getMaster().getClusterStatus(). getServers().size() > NB_SERVERS + || cluster.getLiveRegionServerThreads().size() > NB_SERVERS); } private void awaitDaughters(byte[] tableName, int numDaughters) throws InterruptedException { http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java deleted file mode 100644 index d9e8480..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.hadoop.hbase.LargeTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -/** - * Like {@link TestRegionMergeTransaction} in that we're testing - * {@link RegionMergeTransaction} only the below tests are against a running - * cluster where {@link TestRegionMergeTransaction} is tests against bare - * {@link HRegion}. - */ -@Category(LargeTests.class) -public class TestZKLessMergeOnCluster extends TestRegionMergeTransactionOnCluster { - @BeforeClass - public static void beforeAllTests() throws Exception { - // Don't use ZK for region assignment - TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false); - setupOnce(); - } - - @AfterClass - public static void afterAllTests() throws Exception { - TestRegionMergeTransactionOnCluster.afterAllTests(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/17dff681/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java deleted file mode 100644 index 1e3de64..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import org.apache.hadoop.hbase.LargeTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -/** - * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction} - * only the below tests are against a running cluster where {@link TestSplitTransaction} - * is tests against a bare {@link HRegion}. - */ -@Category(LargeTests.class) -public class TestZKLessSplitOnCluster extends TestSplitTransactionOnCluster { - @BeforeClass - public static void before() throws Exception { - // Don't use ZK for region assignment - TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false); - setupOnce(); - } - - @AfterClass - public static void after() throws Exception { - TestSplitTransactionOnCluster.after(); - } -} -
