This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-22514 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 60e8adc1a5c41e5789b26f64a14ae1c2e1fced6a Author: linkaline <linkalin...@gmail.com> AuthorDate: Fri Aug 16 16:52:41 2019 +0800 HBASE-22820 Do not need to persist default rs group now (#482) Signed-off-by: Duo Zhang <zhang...@apache.org> --- .../hbase/rsgroup/RSGroupInfoManagerImpl.java | 40 +++++++++++++--------- .../hbase/rsgroup/VerifyingRSGroupAdminClient.java | 25 ++++++++++++++ 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index 37f3ce6..eaf23f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -396,11 +396,13 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { } // populate puts - for (RSGroupInfo RSGroupInfo : groupMap.values()) { - RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); - Put p = new Put(Bytes.toBytes(RSGroupInfo.getName())); - p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray()); - mutations.add(p); + for (RSGroupInfo gi : groupMap.values()) { + if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { + RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi); + Put p = new Put(Bytes.toBytes(gi.getName())); + p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray()); + mutations.add(p); + } } if (mutations.size() > 0) { @@ -449,7 +451,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { // Make changes visible after having been persisted to the source of truth resetRSGroupMap(newGroupMap); + saveRSGroupMapToZK(newGroupMap); + updateCacheOfRSGroups(newGroupMap.keySet()); + } + + private void saveRSGroupMapToZK(Map<String, RSGroupInfo> newGroupMap) throws IOException { try { String groupBasePath = ZNodePaths.joinZNode(watcher.getZNodePaths().baseZNode, RS_GROUP_ZNODE); @@ -463,14 +470,16 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { } } - for (RSGroupInfo RSGroupInfo : newGroupMap.values()) { - String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName()); - RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo); - LOG.debug("Updating znode: " + znode); - ZKUtil.createAndFailSilent(watcher, znode); - zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); - zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode, - ProtobufUtil.prependPBMagic(proto.toByteArray()))); + for (RSGroupInfo gi : newGroupMap.values()) { + if (!gi.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { + String znode = ZNodePaths.joinZNode(groupBasePath, gi.getName()); + RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(gi); + LOG.debug("Updating znode: " + znode); + ZKUtil.createAndFailSilent(watcher, znode); + zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode)); + zkOps.add(ZKUtil.ZKUtilOp.createAndFailSilent(znode, + ProtobufUtil.prependPBMagic(proto.toByteArray()))); + } } LOG.debug("Writing ZK GroupInfo count: " + zkOps.size()); @@ -480,7 +489,6 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { masterServices.abort("Failed to write to rsGroupZNode", e); throw new IOException("Failed to write to rsGroupZNode", e); } - updateCacheOfRSGroups(newGroupMap.keySet()); } /** @@ -540,12 +548,12 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { // Called by ServerEventsListenerThread. Synchronize on this because redoing // the rsGroupMap then writing it out. - private synchronized void updateDefaultServers(SortedSet<Address> servers) throws IOException { + private synchronized void updateDefaultServers(SortedSet<Address> servers) { RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP); RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers); HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(newInfo.getName(), newInfo); - flushConfig(newGroupMap); + resetRSGroupMap(newGroupMap); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index a8cd277..28131a9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -26,8 +26,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedSet; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; @@ -133,6 +136,13 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient { tds.addAll(admin.listTableDescriptors()); tds.addAll(admin.listTableDescriptorsByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME)); } + SortedSet<Address> lives = Sets.newTreeSet(); + for (ServerName sn : conn.getAdmin().getClusterMetrics().getLiveServerMetrics().keySet()) { + lives.add(sn.getAddress()); + } + for (ServerName sn : conn.getAdmin().listDecommissionedRegionServers()) { + lives.remove(sn.getAddress()); + } try (Table table = conn.getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); ResultScanner scanner = table.getScanner(new Scan())) { for (;;) { @@ -144,8 +154,22 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient { RSGroupInfoManagerImpl.META_FAMILY_BYTES, RSGroupInfoManagerImpl.META_QUALIFIER_BYTES)); RSGroupInfo rsGroupInfo = ProtobufUtil.toGroupInfo(proto); groupMap.put(proto.getName(), RSGroupUtil.fillTables(rsGroupInfo, tds)); + for(Address address : rsGroupInfo.getServers()){ + lives.remove(address); + } } } + SortedSet<TableName> tables = Sets.newTreeSet(); + for (TableDescriptor td : conn.getAdmin().listTableDescriptors(Pattern.compile(".*"), + true)){ + String groupName = td.getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP); + if (groupName.equals(RSGroupInfo.DEFAULT_GROUP)) { + tables.add(td.getTableName()); + } + } + + groupMap.put(RSGroupInfo.DEFAULT_GROUP, + new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, lives, tables)); assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(wrapped.listRSGroups())); try { String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup"); @@ -160,6 +184,7 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdminClient { zList.add(RSGroupUtil.fillTables(rsGroupInfo, tds)); } } + groupMap.remove(RSGroupInfo.DEFAULT_GROUP); assertEquals(zList.size(), groupMap.size()); for (RSGroupInfo rsGroupInfo : zList) { assertTrue(groupMap.get(rsGroupInfo.getName()).equals(rsGroupInfo));