This is an automated email from the ASF dual-hosted git repository. ndimiduk pushed a commit to branch branch-2.3 in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push: new 9491124 HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count 9491124 is described below commit 949112422fd23297747e441a2163e8f5aa7dce73 Author: Nick Dimiduk <ndimi...@apache.org> AuthorDate: Tue Jun 30 16:50:10 2020 -0700 HBASE-24662 Update DumpClusterStatusAction to notice changes in region server count Sometimes running chaos monkey, I've found that we lose accounting of region servers. I've taken to a manual process of checking the reported list against a known reference. It occurs to me that ChaosMonkey has a known reference, and it can do this accounting for me. Signed-off-by: Viraj Jasani <vjas...@apache.org> --- .../chaos/actions/DumpClusterStatusAction.java | 62 +++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java index dfb506d..d3ad314 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java @@ -19,6 +19,13 @@ package org.apache.hadoop.hbase.chaos.actions; import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.ClusterMetrics; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.net.Address; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,18 +35,69 @@ import org.slf4j.LoggerFactory; public class DumpClusterStatusAction extends Action { private static final Logger LOG = LoggerFactory.getLogger(DumpClusterStatusAction.class); - @Override protected Logger getLogger() { + private Set<Address> initialRegionServers; + + @Override + protected Logger getLogger() { return LOG; } @Override public void init(ActionContext context) throws IOException { super.init(context); + initialRegionServers = collectKnownRegionServers(initialStatus); } @Override public void perform() throws Exception { getLogger().debug("Performing action: Dump cluster status"); - getLogger().info("Cluster status\n" + cluster.getClusterMetrics()); + final ClusterMetrics currentMetrics = cluster.getClusterMetrics(); + getLogger().info("Cluster status\n{}", currentMetrics); + reportMissingRegionServers(currentMetrics); + reportNewRegionServers(currentMetrics); + } + + /** + * Build a set of all the host:port pairs of region servers known to this cluster. + */ + private static Set<Address> collectKnownRegionServers(final ClusterMetrics clusterMetrics) { + final Set<Address> regionServers = clusterMetrics.getLiveServerMetrics() + .keySet() + .stream() + .map(ServerName::getAddress) + .collect(Collectors.toSet()); + clusterMetrics.getDeadServerNames() + .stream() + .map(ServerName::getAddress) + .forEach(regionServers::add); + return Collections.unmodifiableSet(regionServers); + } + + private void reportMissingRegionServers(final ClusterMetrics clusterMetrics) { + final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics); + final Set<Address> missingRegionServers = new HashSet<>(initialRegionServers); + missingRegionServers.removeAll(regionServers); + if (!missingRegionServers.isEmpty()) { + final StringBuilder stringBuilder = new StringBuilder() + .append("region server(s) are missing from this cluster report"); + missingRegionServers.stream() + .sorted() + .forEach(address -> stringBuilder.append("\n ").append(address)); + getLogger().warn(stringBuilder.toString()); + } + } + + private void reportNewRegionServers(final ClusterMetrics clusterMetrics) { + final Set<Address> regionServers = collectKnownRegionServers(clusterMetrics); + final Set<Address> newRegionServers = new HashSet<>(regionServers); + newRegionServers.removeAll(initialRegionServers); + if (!newRegionServers.isEmpty()) { + final StringBuilder stringBuilder = new StringBuilder() + .append("region server(s) are new for this cluster report"); + newRegionServers.stream() + .sorted() + .forEach(address -> stringBuilder.append("\n ").append(address)); + getLogger().warn(stringBuilder.toString()); + } } }