This is an automated email from the ASF dual-hosted git repository. nkalmar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push: new 6af151a ZOOKEEPER-3366: Pluggable metrics system for ZooKeeper - move remaining metrics to MetricsProvider 6af151a is described below commit 6af151a7e44760c33165e58d3b33a4d2760283a4 Author: Enrico Olivelli <eolive...@apache.org> AuthorDate: Tue Jun 4 10:55:09 2019 +0200 ZOOKEEPER-3366: Pluggable metrics system for ZooKeeper - move remaining metrics to MetricsProvider Migrate all remaining metrics to MetricsProvider. We are introducing now *Gauges* which are callbacks to be called when the Provider needs to publish current values, a Gauge is a numeric value that can go up and down. As during the lifecycle of a ZK server process we can have several ZooKeeperServer instances (we have several subclasses), depending on the role of the local peer, sometimes we have to clean up unused Gauges. The old approach in 4lw and on http admin API was to hard code metrics, with multiple 'instanceof' conditions. So we introduce ZooKeeperServer#registerMetrics and ZooKeeperServer#unregisterMetrics: these overridable functions enable each ZooKeeperServer subclass to declare specific Gauges. We are also introducing ZooKeeperServer#collectMonitorValues in order to push non-metrics to monitor commands (admin and 4lw), like "server state" or "version". Author: Enrico Olivelli <eolive...@apache.org> Reviewers: Norbert Kalmar <nkal...@apache.org> Closes #918 from eolivelli/fix/other-metrics --- .../java/org/apache/zookeeper/metrics/Gauge.java | 2 +- .../apache/zookeeper/metrics/MetricsContext.java | 15 ++- .../metrics/impl/DefaultMetricsProvider.java | 27 +++++- .../metrics/impl/NullMetricsProvider.java | 7 +- .../apache/zookeeper/server/ZooKeeperServer.java | 108 ++++++++++++++++++++- .../apache/zookeeper/server/admin/Commands.java | 81 +--------------- .../zookeeper/server/command/MonitorCommand.java | 82 ++++------------ .../server/quorum/FollowerZooKeeperServer.java | 27 ++++++ .../server/quorum/LeaderZooKeeperServer.java | 60 ++++++++++++ .../server/quorum/ObserverZooKeeperServer.java | 8 ++ .../apache/zookeeper/server/quorum/QuorumPeer.java | 15 +++ .../server/quorum/QuorumZooKeeperServer.java | 29 ++++++ .../zookeeper/metrics/BaseTestMetricsProvider.java | 6 +- .../zookeeper/server/admin/CommandsTest.java | 9 ++ .../apache/zookeeper/test/ObserverMasterTest.java | 18 ++-- 15 files changed, 324 insertions(+), 170 deletions(-) diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java index fb8fd67..31ad71b 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java @@ -31,5 +31,5 @@ public interface Gauge { * * @return the current value for the gauge */ - long get(); + Number get(); } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java index ba6de61..c7e7ad0 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java @@ -52,14 +52,23 @@ public interface MetricsContext { /** * Registers an user provided {@link Gauge} which will be called by the * MetricsProvider in order to sample an integer value. + * If another Gauge was already registered the new one will + * take its place. + * Registering a null callback is not allowed. * * @param name unique name of the Gauge in this context * @param gauge the implementation of the Gauge * - * @return true if the Gauge was successfully registered, false if the Gauge - * was already registered. */ - boolean registerGauge(String name, Gauge gauge); + void registerGauge(String name, Gauge gauge); + + /** + * Unregisters the user provided {@link Gauge} bound to the given name. + * + * @param name unique name of the Gauge in this context + * + */ + void unregisterGauge(String name); static enum DetailLevel { /** diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java index c09abc6..9a460a1 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java @@ -17,6 +17,7 @@ */ package org.apache.zookeeper.metrics.impl; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -33,6 +34,8 @@ import org.apache.zookeeper.server.metric.AvgMinMaxCounterSet; import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounter; import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounterSet; import org.apache.zookeeper.server.metric.SimpleCounter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Default implementation of {@link MetricsProvider}.<br> @@ -60,6 +63,8 @@ public class DefaultMetricsProvider implements MetricsProvider { @Override public void stop() { + // release all references to external objects + rootMetricsContext.gauges.clear(); } @Override @@ -74,6 +79,7 @@ public class DefaultMetricsProvider implements MetricsProvider { private static final class DefaultMetricsContext implements MetricsContext { + private final ConcurrentMap<String, Gauge> gauges = new ConcurrentHashMap<>(); private final ConcurrentMap<String, SimpleCounter> counters = new ConcurrentHashMap<>(); private final ConcurrentMap<String, AvgMinMaxCounter> basicSummaries = new ConcurrentHashMap<>(); private final ConcurrentMap<String, AvgMinMaxPercentileCounter> summaries = new ConcurrentHashMap<>(); @@ -82,7 +88,7 @@ public class DefaultMetricsProvider implements MetricsProvider { @Override public MetricsContext getContext(String name) { - // no hierarchy + // no hierarchy yet return this; } @@ -94,9 +100,15 @@ public class DefaultMetricsProvider implements MetricsProvider { } @Override - public boolean registerGauge(String name, Gauge gauge) { - // Not supported - return false; + public void registerGauge(String name, Gauge gauge) { + Objects.requireNonNull(gauge, + "Cannot register a null Gauge for "+name); + gauges.put(name, gauge); + } + + @Override + public void unregisterGauge(String name) { + gauges.remove(name); } @Override @@ -138,6 +150,12 @@ public class DefaultMetricsProvider implements MetricsProvider { } void dump(BiConsumer<String, Object> sink) { + gauges.forEach((name, metric) -> { + Number value = metric.get(); + if (value != null) { + sink.accept(name, value); + } + }); counters.values().forEach(metric -> { metric.values().forEach(sink); }); @@ -171,6 +189,7 @@ public class DefaultMetricsProvider implements MetricsProvider { summarySets.values().forEach(metric -> { metric.reset(); }); + // no need to reset gauges } } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java index 85f01ee..89db0fd 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java @@ -77,8 +77,11 @@ public class NullMetricsProvider implements MetricsProvider { } @Override - public boolean registerGauge(String name, Gauge gauge) { - return true; + public void registerGauge(String name, Gauge gauge) { + } + + @Override + public void unregisterGauge(String name) { } @Override diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java index 6379f1b..5117bd8 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java @@ -36,6 +36,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; import javax.security.sasl.SaslException; @@ -46,6 +47,7 @@ import org.apache.zookeeper.Environment; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.KeeperException.SessionExpiredException; +import org.apache.zookeeper.Version; import org.apache.zookeeper.ZooDefs.OpCode; import org.apache.zookeeper.ZookeeperBanner; import org.apache.zookeeper.common.Time; @@ -53,6 +55,7 @@ import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.data.StatPersisted; import org.apache.zookeeper.jmx.MBeanRegistry; +import org.apache.zookeeper.metrics.MetricsContext; import org.apache.zookeeper.proto.AuthPacket; import org.apache.zookeeper.proto.ConnectRequest; import org.apache.zookeeper.proto.ConnectResponse; @@ -65,12 +68,12 @@ import org.apache.zookeeper.server.RequestProcessor.RequestProcessorException; import org.apache.zookeeper.server.ServerCnxn.CloseRequestException; import org.apache.zookeeper.server.SessionTracker.Session; import org.apache.zookeeper.server.SessionTracker.SessionExpirer; -import org.apache.zookeeper.server.auth.AuthenticationProvider; import org.apache.zookeeper.server.auth.ProviderRegistry; import org.apache.zookeeper.server.auth.ServerAuthenticationProvider; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer; import org.apache.zookeeper.server.util.JvmPauseMonitor; +import org.apache.zookeeper.server.util.OSMXBean; import org.apache.zookeeper.txn.CreateSessionTxn; import org.apache.zookeeper.txn.TxnHeader; import org.slf4j.Logger; @@ -230,6 +233,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { + " clientPortListenBacklog " + getClientPortListenBacklog() + " datadir " + txnLogFactory.getDataDir() + " snapdir " + txnLogFactory.getSnapDir()); + } public String getInitialConfig() { @@ -558,6 +562,8 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { startJvmPauseMonitor(); + registerMetrics(); + setState(State.RUNNING); notifyAll(); } @@ -663,6 +669,11 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { // new RuntimeException("Calling shutdown").printStackTrace(); setState(State.SHUTDOWN); + + // unregister all metrics that are keeping a strong reference to this object + // subclasses will do their specific clean up + unregisterMetrics(); + // Since sessionTracker and syncThreads poll we just have to // set running to false and they will detect it during the poll // interval. @@ -1257,7 +1268,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { // there might be a race condition that it enabled recv after // processing request and then disabled when check throttling. // - // Be aware that we're actually checking the global outstanding + // Be aware that we're actually checking the global outstanding // request before this request. // // It's fine if the IOException thrown before we decrease the count @@ -1437,4 +1448,97 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider { public ResponseCache getReadResponseCache() { return isResponseCachingEnabled ? readResponseCache : null; } + + protected void registerMetrics() { + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + final ZKDatabase zkdb = this.getZKDatabase(); + final ServerStats stats = this.serverStats(); + + rootContext.registerGauge("avg_latency", stats::getAvgLatency); + + rootContext.registerGauge("max_latency", stats::getMaxLatency); + rootContext.registerGauge("min_latency", stats::getMinLatency); + + rootContext.registerGauge("packets_received", stats::getPacketsReceived); + rootContext.registerGauge("packets_sent", stats::getPacketsSent); + rootContext.registerGauge("num_alive_connections", stats::getNumAliveClientConnections); + + rootContext.registerGauge("outstanding_requests", stats::getOutstandingRequests); + rootContext.registerGauge("uptime", stats::getUptime); + + rootContext.registerGauge("znode_count", zkdb::getNodeCount); + + rootContext.registerGauge("watch_count", zkdb.getDataTree()::getWatchCount); + rootContext.registerGauge("ephemerals_count", zkdb.getDataTree()::getEphemeralsCount); + + rootContext.registerGauge("approximate_data_size", zkdb.getDataTree()::cachedApproximateDataSize); + + rootContext.registerGauge("global_sessions", zkdb::getSessionCount); + rootContext.registerGauge("local_sessions", + this.getSessionTracker()::getLocalSessionCount); + + OSMXBean osMbean = new OSMXBean(); + rootContext.registerGauge("open_file_descriptor_count", osMbean::getOpenFileDescriptorCount); + rootContext.registerGauge("max_file_descriptor_count", osMbean::getMaxFileDescriptorCount); + rootContext.registerGauge("connection_drop_probability", this::getConnectionDropChance); + + rootContext.registerGauge("last_client_response_size", stats.getClientResponseStats()::getLastBufferSize); + rootContext.registerGauge("max_client_response_size", stats.getClientResponseStats()::getMaxBufferSize); + rootContext.registerGauge("min_client_response_size", stats.getClientResponseStats()::getMinBufferSize); + + } + + protected void unregisterMetrics() { + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + rootContext.unregisterGauge("avg_latency"); + + rootContext.unregisterGauge("max_latency"); + rootContext.unregisterGauge("min_latency"); + + rootContext.unregisterGauge("packets_received"); + rootContext.unregisterGauge("packets_sent"); + rootContext.unregisterGauge("num_alive_connections"); + + rootContext.unregisterGauge("outstanding_requests"); + rootContext.unregisterGauge("uptime"); + + rootContext.unregisterGauge("znode_count"); + + rootContext.unregisterGauge("watch_count"); + rootContext.unregisterGauge("ephemerals_count"); + rootContext.unregisterGauge("approximate_data_size"); + + rootContext.unregisterGauge("global_sessions"); + rootContext.unregisterGauge("local_sessions"); + + rootContext.unregisterGauge("open_file_descriptor_count"); + rootContext.unregisterGauge("max_file_descriptor_count"); + rootContext.unregisterGauge("connection_drop_probability"); + + rootContext.unregisterGauge("last_client_response_size"); + rootContext.unregisterGauge("max_client_response_size"); + rootContext.unregisterGauge("min_client_response_size"); + + } + + /** + * Hook into admin server, useful to expose additional data + * that do not represent metrics. + * + * @param response a sink which collects the data. + */ + public void dumpMonitorValues(BiConsumer<String, Object> response) { + ServerStats stats = serverStats(); + response.accept("version", Version.getFullVersion()); + response.accept("server_state", stats.getServerState()); + } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java index 4cbb476..cff9c9b 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java @@ -36,7 +36,6 @@ import org.apache.zookeeper.server.DataTree; import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ServerMetrics; import org.apache.zookeeper.server.ServerStats; -import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.ZooTrace; import org.apache.zookeeper.server.persistence.SnapshotInfo; @@ -44,11 +43,8 @@ import org.apache.zookeeper.server.quorum.Follower; import org.apache.zookeeper.server.quorum.FollowerZooKeeperServer; import org.apache.zookeeper.server.quorum.Leader; import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer; -import org.apache.zookeeper.server.quorum.QuorumPeer; -import org.apache.zookeeper.server.quorum.QuorumZooKeeperServer; import org.apache.zookeeper.server.quorum.ObserverZooKeeperServer; import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer; -import org.apache.zookeeper.server.util.OSMXBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -354,83 +350,10 @@ public class Commands { @Override public CommandResponse run(ZooKeeperServer zkServer, Map<String, String> kwargs) { CommandResponse response = initializeResponse(); - - response.put("version", Version.getFullVersion()); - - OSMXBean osMbean = new OSMXBean(); - response.put("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount()); - response.put("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount()); - - if (zkServer != null) { - ZKDatabase zkdb = zkServer.getZKDatabase(); - ServerStats stats = zkServer.serverStats(); - response.put("avg_latency", stats.getAvgLatency()); - response.put("max_latency", stats.getMaxLatency()); - response.put("min_latency", stats.getMinLatency()); - - response.put("packets_received", stats.getPacketsReceived()); - response.put("packets_sent", stats.getPacketsSent()); - response.put("num_alive_connections", stats.getNumAliveClientConnections()); - - response.put("outstanding_requests", stats.getOutstandingRequests()); - response.put("uptime", stats.getUptime()); - - response.put("server_state", stats.getServerState()); - response.put("znode_count", zkdb.getNodeCount()); - - response.put("watch_count", zkdb.getDataTree().getWatchCount()); - response.put("ephemerals_count", zkdb.getDataTree().getEphemeralsCount()); - response.put("approximate_data_size", zkdb.getDataTree().cachedApproximateDataSize()); - - response.put("global_sessions", zkdb.getSessionCount()); - response.put("local_sessions", - zkServer.getSessionTracker().getLocalSessionCount()); - - response.put("connection_drop_probability", zkServer.getConnectionDropChance()); - - response.put("last_client_response_size", stats.getClientResponseStats().getLastBufferSize()); - response.put("max_client_response_size", stats.getClientResponseStats().getMaxBufferSize()); - response.put("min_client_response_size", stats.getClientResponseStats().getMinBufferSize()); - - if (zkServer instanceof QuorumZooKeeperServer) { - QuorumPeer peer = ((QuorumZooKeeperServer) zkServer).self; - response.put("quorum_size", peer.getQuorumSize()); - } - - if (zkServer instanceof LeaderZooKeeperServer) { - Leader leader = ((LeaderZooKeeperServer) zkServer).getLeader(); - - response.put("learners", leader.getLearners().size()); - response.put("synced_followers", leader.getForwardingFollowers().size()); - response.put("synced_non_voting_followers", leader.getNonVotingFollowers().size()); - response.put("synced_observers", leader.getObservingLearners().size()); - response.put("pending_syncs", leader.getNumPendingSyncs()); - response.put("leader_uptime", leader.getUptime()); - - response.put("last_proposal_size", leader.getProposalStats().getLastBufferSize()); - response.put("max_proposal_size", leader.getProposalStats().getMaxBufferSize()); - response.put("min_proposal_size", leader.getProposalStats().getMinBufferSize()); - } - - if (zkServer instanceof FollowerZooKeeperServer) { - Follower follower = ((FollowerZooKeeperServer) zkServer).getFollower(); - Integer syncedObservers = follower.getSyncedObserverSize(); - if (syncedObservers != null) { - response.put("synced_observers", syncedObservers); - } - } - - if (zkServer instanceof ObserverZooKeeperServer) { - response.put("observer_master_id", ((ObserverZooKeeperServer) zkServer).getObserver().getLearnerMasterId()); - } - } - + zkServer.dumpMonitorValues(response::put); ServerMetrics.getMetrics() .getMetricsProvider() - .dump( - (metric, value) -> { - response.put(metric, value); - }); + .dump(response::put); return response; }} diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java index 8cd97da..2718d3c 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java @@ -20,14 +20,8 @@ package org.apache.zookeeper.server.command; import java.io.PrintWriter; -import org.apache.zookeeper.Version; import org.apache.zookeeper.server.ServerCnxn; import org.apache.zookeeper.server.ServerMetrics; -import org.apache.zookeeper.server.ServerStats; -import org.apache.zookeeper.server.ZKDatabase; -import org.apache.zookeeper.server.quorum.Leader; -import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer; -import org.apache.zookeeper.server.util.OSMXBean; public class MonitorCommand extends AbstractFourLetterCommand { @@ -41,73 +35,31 @@ public class MonitorCommand extends AbstractFourLetterCommand { pw.println(ZK_NOT_SERVING); return; } - ZKDatabase zkdb = zkServer.getZKDatabase(); - ServerStats stats = zkServer.serverStats(); - print("version", Version.getFullVersion()); - - print("avg_latency", stats.getAvgLatency()); - print("max_latency", stats.getMaxLatency()); - print("min_latency", stats.getMinLatency()); - - print("packets_received", stats.getPacketsReceived()); - print("packets_sent", stats.getPacketsSent()); - print("num_alive_connections", stats.getNumAliveClientConnections()); - - print("outstanding_requests", stats.getOutstandingRequests()); - - print("server_state", stats.getServerState()); - print("znode_count", zkdb.getNodeCount()); - - print("watch_count", zkdb.getDataTree().getWatchCount()); - print("ephemerals_count", zkdb.getDataTree().getEphemeralsCount()); - print("approximate_data_size", zkdb.getDataTree().cachedApproximateDataSize()); - - OSMXBean osMbean = new OSMXBean(); - if (osMbean != null && osMbean.getUnix() == true) { - print("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount()); - print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount()); - } - - if (stats.getServerState().equals("leader")) { - Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader(); - - print("learners", leader.getLearners().size()); - print("synced_followers", leader.getForwardingFollowers().size()); - print("synced_non_voting_followers", leader.getNonVotingFollowers().size()); - print("pending_syncs", leader.getNumPendingSyncs()); - - print("last_proposal_size", leader.getProposalStats().getLastBufferSize()); - print("max_proposal_size", leader.getProposalStats().getMaxBufferSize()); - print("min_proposal_size", leader.getProposalStats().getMinBufferSize()); - } + // non metrics + zkServer.dumpMonitorValues(this::print); ServerMetrics.getMetrics() .getMetricsProvider() - .dump( - (metric, value) -> { - if (value == null) { - print(metric, null); - } else if (value instanceof Long - || value instanceof Integer) { - print(metric, ((Number) value).longValue()); - } else if (value instanceof Number) { - print(metric, ((Number) value).doubleValue()); - } else { - print(metric, value.toString()); - } - }); - } - - private void print(String key, long number) { - print(key, "" + number); + .dump(this::print); } - private void print(String key, double number) { - print(key, "" + number); + private void print(String key, Object value) { + if (value == null) { + output(key, null); + } else if (value instanceof Long + || value instanceof Integer) { + // format as integers + output(key, value + ""); + } else if (value instanceof Number) { + // format as floating point + output(key, ((Number) value).doubleValue() + ""); + } else { + output(key, value.toString()); + } } - private void print(String key, String value) { + private void output(String key, String value) { pw.print("zk_"); pw.print(key); pw.print("\t"); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java index e8fc15f..1c439f3 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java @@ -37,6 +37,8 @@ import org.apache.zookeeper.server.ServerMetrics; import org.apache.zookeeper.txn.TxnHeader; import javax.management.JMException; +import org.apache.zookeeper.metrics.MetricsContext; +import org.apache.zookeeper.server.ServerMetrics; /** * Just like the standard ZooKeeperServer. We just replace the request @@ -167,4 +169,29 @@ public class FollowerZooKeeperServer extends LearnerZooKeeperServer { } return false; } + + @Override + protected void registerMetrics() { + super.registerMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + rootContext.registerGauge("synced_observers", self::getSynced_observers_metric); + + } + + @Override + protected void unregisterMetrics() { + super.unregisterMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + rootContext.unregisterGauge("synced_observers"); + + } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java index 1833474..74a738f 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java @@ -33,6 +33,8 @@ import org.apache.zookeeper.server.persistence.FileTxnSnapLog; import javax.management.JMException; import java.io.IOException; import java.util.concurrent.TimeUnit; +import org.apache.zookeeper.metrics.MetricsContext; +import org.apache.zookeeper.server.ServerMetrics; /** * @@ -95,6 +97,64 @@ public class LeaderZooKeeperServer extends QuorumZooKeeperServer { } @Override + protected void registerMetrics() { + super.registerMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + rootContext.registerGauge("learners", () -> { + return getLeader().getLearners().size(); + }); + rootContext.registerGauge("synced_followers", () -> { + return getLeader().getForwardingFollowers().size(); + }); + rootContext.registerGauge("synced_non_voting_followers", () -> { + return getLeader().getNonVotingFollowers().size(); + }); + + rootContext.registerGauge("synced_observers", self::getSynced_observers_metric); + + rootContext.registerGauge("pending_syncs", () -> { + return getLeader().getNumPendingSyncs(); + }); + rootContext.registerGauge("leader_uptime", () -> { + return getLeader().getUptime(); + }); + rootContext.registerGauge("last_proposal_size", () -> { + return getLeader().getProposalStats().getLastBufferSize(); + }); + rootContext.registerGauge("max_proposal_size", () -> { + return getLeader().getProposalStats().getMaxBufferSize(); + }); + rootContext.registerGauge("min_proposal_size", () -> { + return getLeader().getProposalStats().getMinBufferSize(); + }); + } + + @Override + protected void unregisterMetrics() { + super.unregisterMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + rootContext.unregisterGauge("learners"); + rootContext.unregisterGauge("synced_followers"); + rootContext.unregisterGauge("synced_non_voting_followers"); + rootContext.unregisterGauge("synced_observers"); + rootContext.unregisterGauge("pending_syncs"); + rootContext.unregisterGauge("leader_uptime"); + + rootContext.unregisterGauge("last_proposal_size"); + rootContext.unregisterGauge("max_proposal_size"); + rootContext.unregisterGauge("min_proposal_size"); + } + + @Override public synchronized void shutdown() { if (containerManager != null) { containerManager.stop(); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java index 951c1fd..e66bd61 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java @@ -19,6 +19,7 @@ package org.apache.zookeeper.server.quorum; import java.io.IOException; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.function.BiConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,4 +142,11 @@ public class ObserverZooKeeperServer extends LearnerZooKeeperServer { syncProcessor.shutdown(); } } + + @Override + public void dumpMonitorValues(BiConsumer<String, Object> response) { + super.dumpMonitorValues(response); + response.accept("observer_master_id", getObserver().getLearnerMasterId()); + } + } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java index c2e45fd..df83df6 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import javax.security.sasl.SaslException; +import org.apache.yetus.audience.InterfaceAudience; import org.apache.zookeeper.KeeperException.BadArgumentsException; import org.apache.zookeeper.common.AtomicFileWritingIdiom; @@ -2336,4 +2337,18 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider Vote vote = getCurrentVote(); return vote != null && id == vote.getId(); } + + @InterfaceAudience.Private + /** + * This is a metric that depends on the status of the peer. + */ + public Integer getSynced_observers_metric() { + if (leader != null) { + return leader.getObservingLearners().size(); + } else if (follower != null) { + return follower.getSyncedObserverSize(); + } else { + return null; + } + } } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java index 72637b2..0e9559a 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java @@ -26,9 +26,11 @@ import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.MultiTransactionRecord; import org.apache.zookeeper.Op; import org.apache.zookeeper.ZooDefs.OpCode; +import org.apache.zookeeper.metrics.MetricsContext; import org.apache.zookeeper.proto.CreateRequest; import org.apache.zookeeper.server.ByteBufferInputStream; import org.apache.zookeeper.server.Request; +import org.apache.zookeeper.server.ServerMetrics; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.ZooKeeperServer; import org.apache.zookeeper.server.persistence.FileTxnSnapLog; @@ -186,4 +188,31 @@ public abstract class QuorumZooKeeperServer extends ZooKeeperServer { protected void setState(State state) { this.state = state; } + + @Override + protected void registerMetrics() { + super.registerMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + rootContext.registerGauge("quorum_size", () -> { + return self.getQuorumSize(); + }); + } + + @Override + protected void unregisterMetrics() { + super.unregisterMetrics(); + + MetricsContext rootContext = ServerMetrics + .getMetrics() + .getMetricsProvider() + .getRootContext(); + + rootContext.unregisterGauge("quorum_size"); + } + } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java b/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java index 7c6e1ae..3e23019 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java @@ -85,10 +85,8 @@ public abstract class BaseTestMetricsProvider implements MetricsProvider { @Override public MetricsContext getRootContext() { - if (!getRootContextCalled.compareAndSet(false, true)) { - // called twice - throw new IllegalStateException(); - } + getRootContextCalled.set(true); + return NullMetricsProvider.NullMetricsContext.INSTANCE; } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java index f8813d0..be803c0 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java @@ -202,6 +202,15 @@ public class CommandsTest extends ClientBase { Map<String, Object> metrics = MetricsUtils.currentServerMetrics(); for (String metric : metrics.keySet()) { + boolean alreadyDefined = fields + .stream() + .anyMatch(f -> { + return f.key.equals(metric); + }); + if (alreadyDefined) { + // known metrics are defined statically in the block above + continue; + } if (metric.startsWith("avg_")) { fields.add(new Field(metric, Double.class)); } else { diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java index 4e07d48..f720634 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java @@ -516,34 +516,32 @@ public class ObserverMasterTest extends QuorumPeerTestBase implements Watcher{ Assert.assertTrue("observer not emitting observer_master_id", stats.containsKey("observer_master_id")); // check the stats for the first peer - stats = Commands.runCommand("mntr", q1.getQuorumPeer().getActiveServer(), emptyMap).toMap(); if (testObserverMaster) { if (q1.getQuorumPeer().leader == null) { - Assert.assertEquals(1, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(1), q1.getQuorumPeer().getSynced_observers_metric()); } else { - Assert.assertEquals(0, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(0), q1.getQuorumPeer().getSynced_observers_metric()); } } else { if (q1.getQuorumPeer().leader == null) { - Assert.assertNull(stats.get("synced_observers")); + Assert.assertNull(q1.getQuorumPeer().getSynced_observers_metric()); } else { - Assert.assertEquals(1, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(1), q1.getQuorumPeer().getSynced_observers_metric()); } } // check the stats for the second peer - stats = Commands.runCommand("mntr", q2.getQuorumPeer().getActiveServer(), emptyMap).toMap(); if (testObserverMaster) { if (q2.getQuorumPeer().leader == null) { - Assert.assertEquals(1, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(1), q2.getQuorumPeer().getSynced_observers_metric()); } else { - Assert.assertEquals(0, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(0), q2.getQuorumPeer().getSynced_observers_metric()); } } else { if (q2.getQuorumPeer().leader == null) { - Assert.assertNull(stats.get("synced_observers")); + Assert.assertNull(q2.getQuorumPeer().getSynced_observers_metric()); } else { - Assert.assertEquals(1, stats.get("synced_observers")); + Assert.assertEquals(Integer.valueOf(1), q2.getQuorumPeer().getSynced_observers_metric()); } }