This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-24950 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 00a042806074e94312946ae8c5956458f6bda9b1 Author: Duo Zhang <zhang...@apache.org> AuthorDate: Wed Sep 9 12:38:32 2020 +0800 HBASE-24606 Implement meta merge (#2311) Signed-off-by: Guanghao Zhang <zg...@apache.org> --- .../hbase/master/assignment/AssignmentManager.java | 2 +- .../hbase/master/assignment/RegionStateStore.java | 8 ++- ...etaSplit.java => TestSimpleMetaSplitMerge.java} | 73 +++++++++++++++++----- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index ab87716..15712e6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -1870,7 +1870,7 @@ public class AssignmentManager { * References removed). */ public void markRegionAsMerged(final RegionInfo child, final ServerName serverName, - RegionInfo [] mergeParents) + RegionInfo[] mergeParents) throws IOException { final RegionStateNode node = regionStates.getOrCreateRegionStateNode(child); node.setState(State.MERGED); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java index 80eafa9..c8bbfa0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java @@ -314,8 +314,12 @@ public class RegionStateStore { private Result getRegionCatalogResult(RegionInfo region) throws IOException { Get get = new Get(CatalogFamilyFormat.getMetaKeyForRegion(region)).addFamily(HConstants.CATALOG_FAMILY); - try (Table table = getMetaTable()) { - return table.get(get); + if (region.isMetaRegion()) { + return masterRegion.get(get); + } else { + try (Table table = getMetaTable()) { + return table.get(get); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java similarity index 59% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java rename to hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java index 110309c..7cbd245 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestSimpleMetaSplitMerge.java @@ -20,17 +20,23 @@ package org.apache.hadoop.hbase; import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -38,11 +44,11 @@ import org.junit.Test; import org.junit.experimental.categories.Category; @Category({ MiscTests.class, MediumTests.class }) -public class TestSimpleMetaSplit { +public class TestSimpleMetaSplitMerge { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestSimpleMetaSplit.class); + HBaseClassTestRule.forClass(TestSimpleMetaSplitMerge.class); private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); @@ -70,29 +76,66 @@ public class TestSimpleMetaSplit { UTIL.shutdownMiniCluster(); } + private void assertMetaRegionCount(int count) { + // do not count it from client as it will reset the location cache for meta table + assertEquals(count, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsOfTable(TableName.META_TABLE_NAME).size()); + } + + private void clearCache(TableName tableName) throws IOException { + try (RegionLocator locator = UTIL.getConnection().getRegionLocator(tableName)) { + locator.clearRegionLocationCache(); + } + } + + private void assertValue(TableName tableName, String row) throws IOException { + try (Table table = UTIL.getConnection().getTable(tableName)) { + Result result = table.get(new Get(Bytes.toBytes(row))); + assertEquals(row, Bytes.toString(result.getValue(CF, CQ))); + } + } + @Test - public void test() throws IOException { + public void test() throws Exception { try (Table table = UTIL.getConnection().getTable(TD1.getTableName())) { table.put(new Put(Bytes.toBytes("row1")).addColumn(CF, CQ, Bytes.toBytes("row1"))); } try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) { table.put(new Put(Bytes.toBytes("row2")).addColumn(CF, CQ, Bytes.toBytes("row2"))); } + Admin admin = UTIL.getAdmin(); // split meta - UTIL.getAdmin().split(TableName.META_TABLE_NAME, Bytes.toBytes("b")); - // do not count it from client as it will reset the location cache for meta table - assertEquals(2, UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() - .getRegionsOfTable(TableName.META_TABLE_NAME).size()); + admin.split(TableName.META_TABLE_NAME, Bytes.toBytes("b")); + assertMetaRegionCount(2); // clear the cache for table 'b' - try (RegionLocator locator = UTIL.getConnection().getRegionLocator(TD2.getTableName())) { - locator.clearRegionLocationCache(); - } + clearCache(TD2.getTableName()); // make sure that we could get the location of the TD2 from the second meta region - try (Table table = UTIL.getConnection().getTable(TD2.getTableName())) { - Result result = table.get(new Get(Bytes.toBytes("row2"))); - assertEquals("row2", Bytes.toString(result.getValue(CF, CQ))); - } + assertValue(TD2.getTableName(), "row2"); // assert from client side - assertEquals(2, UTIL.getAdmin().getRegions(TableName.META_TABLE_NAME).size()); + List<RegionInfo> regions = admin.getRegions(TableName.META_TABLE_NAME); + assertEquals(2, regions.size()); + // compact to make sure we can merge + for (JVMClusterUtil.RegionServerThread t : UTIL.getMiniHBaseCluster() + .getRegionServerThreads()) { + for (HRegion r : t.getRegionServer().getOnlineRegionsLocalContext()) { + if (TableName.isMetaTableName(r.getRegionInfo().getTable())) { + r.compact(true); + for (HStore store : r.getStores()) { + store.closeAndArchiveCompactedFiles(); + } + } + } + } + // merge the 2 regions back to 1 + admin.mergeRegionsAsync(regions.stream().map(RegionInfo::getRegionName).toArray(byte[][]::new), + false).get(); + assertMetaRegionCount(1); + // clear the cache for table 'a' and 'b' + clearCache(TD1.getTableName()); + clearCache(TD2.getTableName()); + + // make sure that we could still get the locations from the new meta region + assertValue(TD2.getTableName(), "row2"); + assertValue(TD1.getTableName(), "row1"); } }