This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 0d11cd4 RATIS-594 RATIS-593 RATIS-551 Add HadoopMetrics2Adaptor for
dropwizard metrics and Graphite container for log service
0d11cd4 is described below
commit 0d11cd47e8b63108e2c3987c23407cd4d4e24da1
Author: Ankit Singhal <[email protected]>
AuthorDate: Mon Jun 17 15:29:03 2019 -0700
RATIS-594 RATIS-593 RATIS-551 Add HadoopMetrics2Adaptor for dropwizard
metrics and Graphite container for log service
Closes #26
Signed-off-by: Josh Elser <[email protected]>
---
LICENSE | 23 +++++
NOTICE | 22 +++++
pom.xml | 26 ++++-
ratis-logservice/docker-compose.yml | 12 +++
ratis-logservice/src/assembly/assembly.xml | 1 +
.../metrics/LogServiceMetricsRegistry.java | 49 +++++----
.../ratis/logservice/server/LogStateMachine.java | 6 +-
.../ratis/logservice/server/MetaStateMachine.java | 6 +-
.../ratis/logservice/LogServiceReadWriteBase.java | 6 +-
.../ratis/logservice/server/TestMetaServer.java | 23 +++--
.../src/test/resources/hadoop-metrics2.properties | 56 +++++++++++
ratis-metrics/pom.xml | 12 +++
.../java/org/apache/ratis/metrics/JVMMetrics.java | 46 +++++++++
.../org/apache/ratis/metrics/MetricRegistries.java | 3 -
.../ratis/metrics/MetricRegistryFactory.java | 3 -
.../apache/ratis/metrics/MetricRegistryInfo.java | 45 ++++++---
.../org/apache/ratis/metrics/MetricsReporting.java | 109 +++++++++++++++++++++
.../apache/ratis/metrics/RatisMetricRegistry.java | 61 ++++++++++++
.../ratis/metrics/impl/MetricRegistriesImpl.java | 2 +-
.../metrics/impl/MetricRegistryFactoryImpl.java | 4 +-
...cRegistry.java => RatisMetricRegistryImpl.java} | 48 ++++++---
.../ratis/metrics/TestHadoop2MetricsAdaptor.java | 75 ++++++++++++++
.../ratis/metrics/TestHadoop2MetricsSink.java} | 36 ++++---
.../src/test/resources/hadoop-metrics2.properties | 44 +++++++++
.../apache/ratis/server/metrics/RatisMetrics.java | 30 ++++--
.../raftlog/segmented/SegmentedRaftLogWorker.java | 3 +-
.../ratis/server/storage/RaftStorageTestUtils.java | 3 +-
.../ratis/server/raftlog/TestRaftLogMetrics.java | 2 +-
28 files changed, 650 insertions(+), 106 deletions(-)
diff --git a/LICENSE b/LICENSE
index ecdafaf..793d5c4 100644
--- a/LICENSE
+++ b/LICENSE
@@ -233,3 +233,26 @@ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR B
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------------------------------------------------------------------------------------------------
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/NOTICE b/NOTICE
index 053c357..e6c5b45 100644
--- a/NOTICE
+++ b/NOTICE
@@ -4,4 +4,26 @@ Copyright 2017-2018 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+--
+This product uses the dropwizard-hadoop-metrics2.
+Copyright 2016 Josh Elser
+
+Licensed under the Apache License v2.0
+
+--
+This product uses https://github.com/mbocek/docker-ganglia/
+
+Contributed by Michal Bocek
+
+Licensed under the Apache License v2.0
+https://github.com/mbocek/docker-ganglia/blob/master/LICENSE
+
+--
+This product uses https://github.com/graphite-project/docker-graphite-statsd
+
+Copyright (c) 2013-2016 Nathan Hopkins
+
+Licensed under the MIT License
+
+--
diff --git a/pom.xml b/pom.xml
index 9ec29e6..082e68f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,6 +158,8 @@
<license-maven-plugin.version>1.19</license-maven-plugin.version>
<copy-rename-maven-plugin.version>1.0</copy-rename-maven-plugin.version>
+
+
<!--
! Can be removed if changing to ASF parent version 19
-->
@@ -193,6 +195,11 @@
<testsThreadCount>4</testsThreadCount>
<checkstyle.version>8.16</checkstyle.version>
+
+ <!--metrics-->
+ <dropwizard.version>3.2.5</dropwizard.version>
+ <dropwizard-metrics-hadoop-metrics2-reporter.version>0.1.2
+ </dropwizard-metrics-hadoop-metrics2-reporter.version>
</properties>
<dependencyManagement>
@@ -304,6 +311,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.joshelser</groupId>
+ <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+
<version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version>
+ </dependency>
<dependency>
<artifactId>ratis-server</artifactId>
@@ -377,7 +389,19 @@
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
- <version>3.2.5</version>
+ <version>${dropwizard.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ <version>${dropwizard.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-ganglia</artifactId>
+ <version>${dropwizard.version}</version>
</dependency>
<dependency>
diff --git a/ratis-logservice/docker-compose.yml
b/ratis-logservice/docker-compose.yml
index 6d068f8..3d7b2a1 100644
--- a/ratis-logservice/docker-compose.yml
+++ b/ratis-logservice/docker-compose.yml
@@ -73,3 +73,15 @@ services:
default:
aliases:
- worker3.logservice.ratis.org
+ graphite:
+ image: graphiteapp/graphite-statsd
+ ports:
+ - 8000:80
+ - 2003-2004:2003-2004
+ - 2023-2024:2023-2024
+ - 8125:8125
+ - 8126:8126
+ networks:
+ default:
+ aliases:
+ - graphite.logservice.ratis.org
diff --git a/ratis-logservice/src/assembly/assembly.xml
b/ratis-logservice/src/assembly/assembly.xml
index 7f287d8..21154bf 100644
--- a/ratis-logservice/src/assembly/assembly.xml
+++ b/ratis-logservice/src/assembly/assembly.xml
@@ -40,6 +40,7 @@
<outputDirectory>conf</outputDirectory>
<includes>
<include>log4j.properties</include>
+ <include>hadoop-metrics2.properties</include>
</includes>
</fileSet>
<fileSet>
diff --git
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetricsRegistry.java
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetricsRegistry.java
index 8e6acfb..08314f2 100644
---
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetricsRegistry.java
+++
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetricsRegistry.java
@@ -18,51 +18,50 @@
package org.apache.ratis.logservice.metrics;
-import java.util.Optional;
+import java.util.concurrent.TimeUnit;
-import com.codahale.metrics.JmxReporter;
import org.apache.ratis.metrics.MetricRegistries;
import org.apache.ratis.metrics.MetricRegistryInfo;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+import org.apache.ratis.metrics.MetricsReporting;
+import org.apache.ratis.metrics.RatisMetricRegistry;
public class LogServiceMetricsRegistry {
- public static final String RATIS_LOG_SERVICE_METRICS_CONTEXT =
"RatisLogService";
+ public static final String RATIS_LOG_STATEMACHINE_METRICS =
"log_statemachine";
+ public static final String RATIS_LOG_SERVICE_METRICS = "ratis_log_service";
public static final String RATIS_LOG_SERVICE_METRICS_DESC = "Ratis log
service metrics";
- public static final String RATIS_LOG_SERVICE_META_DATA_METRICS_CONTEXT =
- "RatisLogServiceMetaData";
+ public static final String RATIS_LOG_SERVICE_META_DATA_METRICS =
"metadata_statemachine";
public static final String RATIS_LOG_SERVICE_META_DATA_METRICS_DESC =
"Ratis log service metadata metrics";
- public static final String JMX_DOMAIN = "RatisLogService";
+ static MetricsReporting metricsReporting = new MetricsReporting(500,
TimeUnit.MILLISECONDS);
- public static RatisMetricRegistry createMetricRegistryForLogService(String
logName) {
- return create(new MetricRegistryInfo(logName,
RATIS_LOG_SERVICE_METRICS_DESC,
- RATIS_LOG_SERVICE_METRICS_CONTEXT));
+ public static RatisMetricRegistry createMetricRegistryForLogService(String
logName,
+ String serverId) {
+ return create(new MetricRegistryInfo(logName + "." + serverId,
RATIS_LOG_SERVICE_METRICS,
+ RATIS_LOG_STATEMACHINE_METRICS, RATIS_LOG_SERVICE_METRICS_DESC));
}
- public static RatisMetricRegistry getMetricRegistryForLogService(String
logName) {
+ public static RatisMetricRegistry getMetricRegistryForLogService(String
logName,
+ String serverId) {
return MetricRegistries.global().get(
- new MetricRegistryInfo(logName, RATIS_LOG_SERVICE_METRICS_DESC,
- RATIS_LOG_SERVICE_METRICS_CONTEXT)).get();
+ new MetricRegistryInfo(logName + "." + serverId,
RATIS_LOG_SERVICE_METRICS,
+ RATIS_LOG_STATEMACHINE_METRICS,
RATIS_LOG_SERVICE_METRICS_DESC)).get();
}
- public static RatisMetricRegistry
createMetricRegistryForLogServiceMetaData(String className) {
- return create(new MetricRegistryInfo(className,
RATIS_LOG_SERVICE_META_DATA_METRICS_DESC,
- RATIS_LOG_SERVICE_META_DATA_METRICS_CONTEXT));
+ public static RatisMetricRegistry
createMetricRegistryForLogServiceMetaData(String serverId) {
+ return create(new MetricRegistryInfo(serverId, RATIS_LOG_SERVICE_METRICS,
+ RATIS_LOG_SERVICE_META_DATA_METRICS,
RATIS_LOG_SERVICE_META_DATA_METRICS_DESC));
}
- public static RatisMetricRegistry
getMetricRegistryForLogServiceMetaData(String className) {
- return MetricRegistries.global().get(
- new MetricRegistryInfo(className,
RATIS_LOG_SERVICE_META_DATA_METRICS_DESC,
- RATIS_LOG_SERVICE_META_DATA_METRICS_CONTEXT)).get();
+ public static RatisMetricRegistry
getMetricRegistryForLogServiceMetaData(String serverId) {
+ return MetricRegistries.global().get(new MetricRegistryInfo(serverId,
RATIS_LOG_SERVICE_METRICS,
+ RATIS_LOG_SERVICE_META_DATA_METRICS,
RATIS_LOG_SERVICE_META_DATA_METRICS_DESC)).get();
}
private static RatisMetricRegistry create(MetricRegistryInfo info) {
- Optional<RatisMetricRegistry> metricRegistry =
MetricRegistries.global().get(info);
- if (metricRegistry.isPresent()) {
- return metricRegistry.get();
- }
RatisMetricRegistry registry = MetricRegistries.global().create(info);
-
JmxReporter.forRegistry(registry.getDropWizardMetricRegistry()).inDomain(JMX_DOMAIN).build().start();
+ metricsReporting
+ .startMetricsReporter(registry,
MetricsReporting.MetricReporterType.JMX,
+ MetricsReporting.MetricReporterType.HADOOP2);
return registry;
}
diff --git
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
index 7405daa..9185641 100644
---
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
+++
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
@@ -42,7 +42,7 @@ import
org.apache.ratis.logservice.proto.LogServiceProtos.GetStateRequestProto;
import
org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceRequestProto;
import org.apache.ratis.logservice.proto.LogServiceProtos.ReadLogRequestProto;
import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+import org.apache.ratis.metrics.RatisMetricRegistry;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.proto.RaftProtos.LogEntryProto;
import org.apache.ratis.protocol.Message;
@@ -128,8 +128,8 @@ public class LogStateMachine extends BaseStateMachine {
this.proxy = (RaftServerProxy) server;
this.groupId = groupId;
//TODO: using groupId for metric now but better to tag it with LogName
- this.metricRegistry =
-
LogServiceMetricsRegistry.createMetricRegistryForLogService(groupId.toString());
+ this.metricRegistry = LogServiceMetricsRegistry
+ .createMetricRegistryForLogService(groupId.toString(),
server.getId().toString());
this.readNextQueryTimer = metricRegistry.timer("readNextQueryTime");
this.startIndexTimer= metricRegistry.timer("startIndexTime");
this.sizeRequestTimer = metricRegistry.timer("sizeRequestTime");
diff --git
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
index 3a255ad..7caa63a 100644
---
a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
+++
b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
@@ -32,7 +32,6 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
-import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
@@ -53,7 +52,8 @@ import
org.apache.ratis.logservice.proto.MetaServiceProtos.LogServiceUnregisterL
import org.apache.ratis.logservice.proto.MetaServiceProtos.MetaSMRequestProto;
import org.apache.ratis.logservice.util.LogServiceProtoUtil;
import org.apache.ratis.logservice.util.MetaServiceProtoUtil;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+import org.apache.ratis.metrics.RatisMetricRegistry;
+import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.protocol.ClientId;
import org.apache.ratis.protocol.Message;
@@ -115,7 +115,7 @@ public class MetaStateMachine extends BaseStateMachine {
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage
storage) throws IOException {
this.raftServer = server;
this.metricRegistry = LogServiceMetricsRegistry
-
.createMetricRegistryForLogServiceMetaData(getClass().getSimpleName());
+
.createMetricRegistryForLogServiceMetaData(server.getId().toString());
super.initialize(server, groupId, storage);
}
diff --git
a/ratis-logservice/src/test/java/org/apache/ratis/logservice/LogServiceReadWriteBase.java
b/ratis-logservice/src/test/java/org/apache/ratis/logservice/LogServiceReadWriteBase.java
index 166702e..925ebac 100644
---
a/ratis-logservice/src/test/java/org/apache/ratis/logservice/LogServiceReadWriteBase.java
+++
b/ratis-logservice/src/test/java/org/apache/ratis/logservice/LogServiceReadWriteBase.java
@@ -314,8 +314,10 @@ public abstract class LogServiceReadWriteBase<CLUSTER
extends MiniRaftCluster>
}
private Long getJMXCount(String groupId, String metricName) throws Exception
{
- ObjectName oname = new ObjectName(LogServiceMetricsRegistry.JMX_DOMAIN,
"name",
- groupId + "." +
LogServiceMetricsRegistry.RATIS_LOG_SERVICE_METRICS_CONTEXT + "." + metricName);
+ ObjectName oname = new
ObjectName(LogServiceMetricsRegistry.RATIS_LOG_SERVICE_METRICS, "name",
+ LogServiceMetricsRegistry
+ .getMetricRegistryForLogService(groupId,
cluster.getLeader().getId().toString())
+ .getMetricRegistryInfo().getName() + "." + metricName);
return (Long)
ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count");
}
}
diff --git
a/ratis-logservice/src/test/java/org/apache/ratis/logservice/server/TestMetaServer.java
b/ratis-logservice/src/test/java/org/apache/ratis/logservice/server/TestMetaServer.java
index 7dab2f5..685cf9b 100644
---
a/ratis-logservice/src/test/java/org/apache/ratis/logservice/server/TestMetaServer.java
+++
b/ratis-logservice/src/test/java/org/apache/ratis/logservice/server/TestMetaServer.java
@@ -20,6 +20,7 @@ package org.apache.ratis.logservice.server;
import org.apache.ratis.logservice.api.*;
import org.apache.ratis.logservice.client.LogServiceClient;
+import org.apache.ratis.logservice.common.Constants;
import org.apache.ratis.logservice.common.LogAlreadyExistException;
import org.apache.ratis.logservice.common.LogNotFoundException;
import org.apache.ratis.logservice.metrics.LogServiceMetricsRegistry;
@@ -36,6 +37,7 @@ import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
import java.util.List;
+import java.util.NoSuchElementException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;
@@ -44,6 +46,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
+import javax.management.InstanceNotFoundException;
import javax.management.ObjectName;
public class TestMetaServer {
@@ -202,12 +205,20 @@ public class TestMetaServer {
assertEquals(expectedCount, getJMXCount(metricName));
}
- private Long getJMXCount(String metricName) throws Exception{
- ObjectName oname = new
ObjectName(LogServiceMetricsRegistry.JMX_DOMAIN, "name",
- MetaStateMachine.class.getSimpleName() + "."
- +
LogServiceMetricsRegistry.RATIS_LOG_SERVICE_META_DATA_METRICS_CONTEXT + "."
- + metricName);
- return (Long)
ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count");
+ private Long getJMXCount(String metricName) throws Exception {
+ for (MetadataServer master : cluster.getMasters()) {
+ ObjectName oname =
+ new
ObjectName(LogServiceMetricsRegistry.RATIS_LOG_SERVICE_METRICS, "name",
+
LogServiceMetricsRegistry.getMetricRegistryForLogServiceMetaData(master.getId())
+ .getMetricRegistryInfo().getName() + "." + metricName);
+ try {
+ return (Long) ManagementFactory.getPlatformMBeanServer()
+ .getAttribute(oname, "Count");
+ } catch (InstanceNotFoundException e) {
+
+ }
+ }
+ throw new InstanceNotFoundException();
}
@Ignore ("Too heavy for the current implementation")
diff --git a/ratis-logservice/src/test/resources/hadoop-metrics2.properties
b/ratis-logservice/src/test/resources/hadoop-metrics2.properties
new file mode 100644
index 0000000..b2a53ac
--- /dev/null
+++ b/ratis-logservice/src/test/resources/hadoop-metrics2.properties
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#There are two options with file names:
+# 1. hadoop-metrics2-[prefix].properties
+# 2. hadoop-metrics2.properties
+# Either will be loaded by the metrics system (but not both).
+#
+# NOTE: The metrics system is only initialized once per JVM (but does
ref-counting, so we can't
+#shutdown and restart), so we only load the first prefix that we find.
Generally, this will be
+# ratis_log_service_metadata/ratis_log_service/ratis (unless someone else
registers first, but for many clients, there should only be one).
+#
+# Usually, you would use hadoop-metrics2-phoenix.properties, but we use the
generic
+# hadoop-metrics2.properties to ensure it these are loaded regardless of where
we are running,
+# assuming there isn't another config on the classpath.
+
+#
+# When specifying sinks, the syntax to use is:
+# [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for detail
+ratis.source.start_mbeans=true
+#*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
+#ratis.sink.ganglia.servers=ganglia.logservice.ratis.org:8649
+#ratis_log_service_metadata.sink.ganglia.servers=ganglia.logservice.ratis.org:8649
+#ratis_log_service.sink.ganglia.servers=ganglia.logservice.ratis.org:8649
+
+*.sink.graphite.class=org.apache.hadoop.metrics2.sink.GraphiteSink
+ratis.sink.graphite.server_host=graphite.logservice.ratis.org
+ratis.sink.graphite.server_port=2003
+ratis.sink.graphite.metrics_prefix=ratis
+
+*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
+ratis.sink.file.filename=/tmp/ratis-metrics.out
+
+
+
+#ratis_log_service.sink.graphite.server_host=graphite.logservice.ratis.org
+#ratis_log_service.sink.graphite.server_port=2003
+#ratis_log_service.sink.graphite.metrics_prefix=ratis_log_Service
+# Frequency, in seconds, of sampling from the sources
+*.period=10
diff --git a/ratis-metrics/pom.xml b/ratis-metrics/pom.xml
index fde261b..75dafa2 100644
--- a/ratis-metrics/pom.xml
+++ b/ratis-metrics/pom.xml
@@ -56,5 +56,17 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>com.github.joshelser</groupId>
+ <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-ganglia</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/JVMMetrics.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/JVMMetrics.java
new file mode 100644
index 0000000..ba387fc
--- /dev/null
+++ b/ratis-metrics/src/main/java/org/apache/ratis/metrics/JVMMetrics.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+
+public class JVMMetrics {
+ static MetricRegistryInfo info = new MetricRegistryInfo("jvm", "ratis_jvm",
"jvm", "jvm metrics");
+ static RatisMetricRegistry registry = MetricRegistries.global().create(info);
+ static {
+ registry.registerAll("gc", new GarbageCollectorMetricSet());
+ registry.registerAll("memory", new MemoryUsageGaugeSet());
+ registry.registerAll("threads", new ThreadStatesGaugeSet());
+ registry.registerAll("classLoading", new ClassLoadingGaugeSet());
+ }
+
+ public static RatisMetricRegistry getRegistry() {
+ return registry;
+ }
+
+ public static void startJVMReporting(long period, TimeUnit unit,
+ MetricsReporting.MetricReporterType... reporting){
+ MetricsReporting metricsReporting = new MetricsReporting(period,unit);
+ metricsReporting.startMetricsReporter(getRegistry(), reporting);
+ }
+}
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistries.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistries.java
index 97c81cf..a9ea10e 100644
--- a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistries.java
+++ b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistries.java
@@ -23,9 +23,6 @@ import java.util.Collection;
import java.util.Optional;
import java.util.Set;
-import com.codahale.metrics.MetricRegistry;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
-
/**
* MetricRegistries is collection of MetricRegistry's. MetricsRegistries
implementations should do
* ref-counting of MetricRegistry's via create() and remove() methods.
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
index 68d3410..8f45deb 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
@@ -20,9 +20,6 @@
package org.apache.ratis.metrics;
-import com.codahale.metrics.MetricRegistry;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
-
/**
* A Factory for creating MetricRegistries. This is the main plugin point for
metrics implementation
*/
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryInfo.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryInfo.java
index e96bb8b..6b6435f 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryInfo.java
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryInfo.java
@@ -30,27 +30,38 @@ import com.codahale.metrics.MetricRegistry;
*/
public class MetricRegistryInfo {
- protected final String metricsName;
+ protected final String prefix;
protected final String metricsDescription;
- protected final String metricsContext;
+ protected final String metricsComponentName;
private final String fullName;
+ private final String applicationName;
- public MetricRegistryInfo(
- String metricsName,
- String metricsDescription,
- String metricsContext) {
- this.metricsName = metricsName;
+ /**
+ * @param prefix className or component name this metric registry collects
metric for
+ * @param applicationName application Name needs to be in small case as it
is used for hadoop2metrics
+ * @param metricsComponentName component name needs to be in small case as
it is used for hadoop2metrics
+ * @param metricsDescription description of the metrics collected by this
registry
+ *
+ */
+ public MetricRegistryInfo(String prefix, String applicationName, String
metricsComponentName,
+ String metricsDescription) {
+ this.prefix = prefix;
+ this.applicationName = applicationName;
+ this.metricsComponentName = metricsComponentName;
this.metricsDescription = metricsDescription;
- this.metricsContext = metricsContext;
- this.fullName = MetricRegistry.name(metricsName,metricsContext);
+ this.fullName = MetricRegistry.name(applicationName, metricsComponentName,
prefix);
+ }
+
+ public String getApplicationName() {
+ return this.applicationName;
}
/**
*
- * @return The string context
+ * @return component name for which Metric is getting collected
*/
- public String getMetricsContext() {
- return metricsContext;
+ public String getMetricsComponentName() {
+ return metricsComponentName;
}
/**
@@ -61,10 +72,10 @@ public class MetricRegistryInfo {
}
/**
- * Get the name of the metrics that are being exported by this registry.
+ * Get the unique prefix for metrics that are being exported by this
registry.
*/
- public String getMetricsName() {
- return metricsName;
+ public String getPrefix() {
+ return prefix;
}
@Override
@@ -78,10 +89,12 @@ public class MetricRegistryInfo {
@Override
public int hashCode() {
- return Objects.hash(metricsName, metricsDescription, metricsContext);
+ return Objects.hash(prefix, metricsDescription, metricsComponentName);
}
public String getName() {
return fullName;
}
+
+
}
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricsReporting.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricsReporting.java
new file mode 100644
index 0000000..666bfc2
--- /dev/null
+++ b/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricsReporting.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.metrics;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ganglia.GangliaReporter;
+import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter;
+import info.ganglia.gmetric4j.gmetric.GMetric;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetricsReporting {
+ public static final Logger LOG =
LoggerFactory.getLogger(MetricsReporting.class);
+ private TimeUnit unit;
+ private long period;
+ private static RatisMetricRegistry jvmRegistry;
+ private GMetric ganglia;
+ static{
+ DefaultMetricsSystem.initialize("ratis");
+ }
+
+ public enum MetricReporterType {
+ JMX, HADOOP2, CONSOLE, GANGLIA
+ }
+
+ public MetricsReporting(long period, TimeUnit unit) {
+ this.period = period;
+ this.unit = unit;
+ }
+
+ public void configureGanglia(String host, int port) throws IOException {
+ ganglia = new GMetric(host, port, GMetric.UDPAddressingMode.MULTICAST, 1);
+
+ }
+
+ /**
+ * @param registry
+ * @param reporting
+ * @return
+ */
+ public boolean startMetricsReporter(RatisMetricRegistry registry,
MetricReporterType... reporting) {
+
+ MetricRegistry dropWizardRegistry = registry.getDropWizardMetricRegistry();
+ for (MetricReporterType reporter : reporting) {
+ try {
+ switch (reporter) {
+ case CONSOLE:
+ ConsoleReporter.forRegistry(dropWizardRegistry)
+
.convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build()
+ .start(period, unit);
+ break;
+ case JMX:
+ JmxReporter.Builder builder =
+ JmxReporter.forRegistry(dropWizardRegistry);
+
builder.inDomain(registry.getMetricRegistryInfo().getApplicationName());
+ builder.build().start();
+ break;
+ case HADOOP2:
+ MetricRegistryInfo info = registry.getMetricRegistryInfo();
+ HadoopMetrics2Reporter.forRegistry(dropWizardRegistry)
+ .build(DefaultMetricsSystem.instance(), info.getName(),
info.getMetricsDescription(),
+ info.getMetricsComponentName()).start(period, unit);
+ break;
+ case GANGLIA:
+ if (ganglia == null) {
+ throw new IllegalStateException(
+ "Ganglia is not configured!! please configure it first by "
+ + "using MetricReporting#configureGanglia..");
+ }
+ GangliaReporter.forRegistry(dropWizardRegistry)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(ganglia);
+ break;
+ default:
+ LOG.warn("Unhandled reporter, " + reporter + " provided.");
+ return false;
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
\ No newline at end of file
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/RatisMetricRegistry.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/RatisMetricRegistry.java
new file mode 100644
index 0000000..b16c6c1
--- /dev/null
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/RatisMetricRegistry.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.metrics;
+
+import java.util.SortedMap;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.Timer;
+import
org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting;
+
+public interface RatisMetricRegistry {
+ Timer timer(String name);
+
+ Counter counter(String name);
+
+ Gauge gauge(String name, MetricRegistry.MetricSupplier<Gauge> supplier);
+
+ Timer timer(String name, MetricRegistry.MetricSupplier<Timer> supplier);
+
+ SortedMap<String, Gauge> getGauges(MetricFilter filter);
+
+ Counter counter(String name, MetricRegistry.MetricSupplier<Counter>
supplier);
+
+ Histogram histogram(String name);
+
+ Meter meter(String name);
+
+ Meter meter(String name, MetricRegistry.MetricSupplier<Meter> supplier);
+
+ @VisibleForTesting Metric get(String shortName);
+
+ <T extends Metric> T register(String name, T metric) throws
IllegalArgumentException;
+
+ MetricRegistry getDropWizardMetricRegistry();
+
+ MetricRegistryInfo getMetricRegistryInfo();
+
+ void registerAll(String prefix, MetricSet metricSet);
+}
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java
index e43f7b9..ec3df20 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java
@@ -23,10 +23,10 @@ import java.util.Collections;
import java.util.Optional;
import java.util.Set;
-import com.codahale.metrics.MetricRegistry;
import org.apache.ratis.metrics.MetricRegistries;
import org.apache.ratis.metrics.MetricRegistryFactory;
import org.apache.ratis.metrics.MetricRegistryInfo;
+import org.apache.ratis.metrics.RatisMetricRegistry;
/**
* Implementation of MetricRegistries that does ref-counting.
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistryFactoryImpl.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistryFactoryImpl.java
index 622f1a7..822f865 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistryFactoryImpl.java
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/MetricRegistryFactoryImpl.java
@@ -18,13 +18,13 @@
*/
package org.apache.ratis.metrics.impl;
-import com.codahale.metrics.MetricRegistry;
import org.apache.ratis.metrics.MetricRegistryFactory;
import org.apache.ratis.metrics.MetricRegistryInfo;
+import org.apache.ratis.metrics.RatisMetricRegistry;
public class MetricRegistryFactoryImpl implements MetricRegistryFactory {
@Override
public RatisMetricRegistry create(MetricRegistryInfo info) {
- return new RatisMetricRegistry(info);
+ return new RatisMetricRegistryImpl(info);
}
}
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistry.java
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistryImpl.java
similarity index 61%
rename from
ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistry.java
rename to
ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistryImpl.java
index 41d4ad3..69fedf4 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistry.java
+++
b/ratis-metrics/src/main/java/org/apache/ratis/metrics/impl/RatisMetricRegistryImpl.java
@@ -17,9 +17,8 @@
*/
package org.apache.ratis.metrics.impl;
+import java.util.Map;
import java.util.SortedMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
@@ -31,59 +30,66 @@ import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.MetricRegistry.MetricSupplier;
import com.codahale.metrics.MetricSet;
import com.codahale.metrics.Timer;
+import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import org.apache.ratis.metrics.MetricRegistryInfo;
+import org.apache.ratis.metrics.RatisMetricRegistry;
import
org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* Custom implementation of {@link MetricRegistry}.
*/
-public class RatisMetricRegistry{
+public class RatisMetricRegistryImpl implements RatisMetricRegistry {
MetricRegistry metricRegistry = new MetricRegistry();
private final MetricRegistryInfo info;
- public RatisMetricRegistry(MetricRegistryInfo info) {
+ public RatisMetricRegistryImpl(MetricRegistryInfo info) {
super();
this.info = info;
}
+ @Override
public Timer timer(String name) {
return metricRegistry.timer(getMetricName(name));
}
+ @Override
public Counter counter(String name) {
return metricRegistry.counter(getMetricName(name));
}
- public Gauge gauge(String name, MetricSupplier<Gauge> supplier) {
+ @Override public Gauge gauge(String name, MetricSupplier<Gauge> supplier) {
return metricRegistry.gauge(getMetricName(name), supplier);
}
- public Timer timer(String name, MetricSupplier<Timer> supplier) {
+ @Override public Timer timer(String name, MetricSupplier<Timer> supplier) {
return metricRegistry.timer(getMetricName(name), supplier);
}
- public SortedMap<String, Gauge> getGauges(MetricFilter filter) {
+ @Override public SortedMap<String, Gauge> getGauges(MetricFilter filter) {
return metricRegistry.getGauges(filter);
}
- public Counter counter(String name, MetricSupplier<Counter> supplier) {
+ @Override public Counter counter(String name, MetricSupplier<Counter>
supplier) {
return metricRegistry.counter(getMetricName(name), supplier);
}
- public Histogram histogram(String name) {
+ @Override public Histogram histogram(String name) {
return metricRegistry.histogram(getMetricName(name));
}
- public Meter meter(String name) {
+ @Override public Meter meter(String name) {
return metricRegistry.meter(getMetricName(name));
}
- public Meter meter(String name, MetricSupplier<Meter> supplier) {
+ @Override public Meter meter(String name, MetricSupplier<Meter> supplier) {
return metricRegistry.meter(getMetricName(name), supplier);
}
- @VisibleForTesting
+ @Override @VisibleForTesting
public Metric get(String shortName) {
return metricRegistry.getMetrics().get(getMetricName(shortName));
}
@@ -92,12 +98,26 @@ public class RatisMetricRegistry{
return MetricRegistry.name(info.getName(), shortName);
}
- public <T extends Metric> T register(String name, T metric) throws
IllegalArgumentException {
+ @Override public <T extends Metric> T register(String name, T metric) throws
IllegalArgumentException {
return metricRegistry.register(getMetricName(name), metric);
}
- public MetricRegistry getDropWizardMetricRegistry() {
+ @Override public MetricRegistry getDropWizardMetricRegistry() {
return metricRegistry;
}
+
+ @Override public MetricRegistryInfo getMetricRegistryInfo(){
+ return this.info;
+ }
+
+ @Override public void registerAll(String prefix, MetricSet metricSet) {
+ for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) {
+ if (entry.getValue() instanceof MetricSet) {
+ registerAll(prefix + "." + entry.getKey(), (MetricSet)
entry.getValue());
+ } else {
+ register(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ }
+ }
}
diff --git
a/ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsAdaptor.java
b/ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsAdaptor.java
new file mode 100644
index 0000000..99381ed
--- /dev/null
+++
b/ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsAdaptor.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.metrics;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Counter;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.junit.Test;
+
+public class TestHadoop2MetricsAdaptor {
+
+ @Test public void testMetrics() throws InterruptedException {
+ MetricRegistryInfo info =
+ new MetricRegistryInfo(TestHadoop2MetricsAdaptor.class.getName(),
"ratis_test",
+ "test","ratis test metrics");
+ RatisMetricRegistry registry = MetricRegistries.global().create(info);
+ MetricsReporting metricsReporting = new MetricsReporting(500,
TimeUnit.MILLISECONDS);
+ metricsReporting
+ .startMetricsReporter(registry,
MetricsReporting.MetricReporterType.HADOOP2);
+ Counter counter = registry.counter("test");
+ counter.inc();
+ counter.inc();
+ counter.inc();
+ counter.dec();
+ int count = 0;
+ Map<String, Long> expectedMetrics = new HashMap<String, Long>();
+
expectedMetrics.put("org.apache.ratis.metrics.TestHadoop2MetricsAdaptor.ratis_test.test",
2L);
+ Thread.sleep(1000);
+ boolean result=false;
+ while ( count < 10) {
+ if(TestHadoop2MetricsSink.metrics != null) {
+ result = verifyMetric(expectedMetrics);
+ if (result) {
+ break;
+ }
+ }
+ Thread.sleep(1000);
+ count++;
+ } assertTrue(result);
+ }
+
+ public boolean verifyMetric(Map<String, Long> expectedMetrics) {
+ for (AbstractMetric metric : TestHadoop2MetricsSink.metrics) {
+ if (expectedMetrics.containsKey(metric.name())) {
+ long expectedValue = expectedMetrics.get(metric.name());
+ long actualValue = metric.value().longValue();
+ if (expectedValue != actualValue) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+}
diff --git
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
b/ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsSink.java
similarity index 50%
copy from
ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
copy to
ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsSink.java
index 68d3410..270b227 100644
---
a/ratis-metrics/src/main/java/org/apache/ratis/metrics/MetricRegistryFactory.java
+++
b/ratis-metrics/src/test/java/org/apache/ratis/metrics/TestHadoop2MetricsSink.java
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -8,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,21 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.ratis.metrics;
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
-package org.apache.ratis.metrics;
+public class TestHadoop2MetricsSink implements MetricsSink {
-import com.codahale.metrics.MetricRegistry;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+ public static final String RATIS_METRICS_RECORD_NAME = "test";
+ static Object lock = new Object();
+ static Iterable<AbstractMetric> metrics;
-/**
- * A Factory for creating MetricRegistries. This is the main plugin point for
metrics implementation
- */
-public interface MetricRegistryFactory {
- /**
- * Create a MetricRegistry from the given MetricRegistryInfo
- * @param info the descriptor for MetricRegistry
- * @return a MetricRegistry implementation
- */
- RatisMetricRegistry create(MetricRegistryInfo info);
+ @Override public void putMetrics(MetricsRecord metricsRecord) {
+ if (metricsRecord.name().equals(RATIS_METRICS_RECORD_NAME)) {
+ TestHadoop2MetricsSink.metrics = metricsRecord.metrics();
+ }
+ }
+
+ @Override public void flush() {
+ }
+
+ @Override public void init(SubsetConfiguration subsetConfiguration) {
+ }
}
diff --git a/ratis-metrics/src/test/resources/hadoop-metrics2.properties
b/ratis-metrics/src/test/resources/hadoop-metrics2.properties
new file mode 100644
index 0000000..3482888
--- /dev/null
+++ b/ratis-metrics/src/test/resources/hadoop-metrics2.properties
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#There are two options with file names:
+# 1. hadoop-metrics2-[prefix].properties
+# 2. hadoop-metrics2.properties
+# Either will be loaded by the metrics system (but not both).
+#
+# NOTE: The metrics system is only initialized once per JVM (but does
ref-counting, so we can't
+#shutdown and restart), so we only load the first prefix that we find.
Generally, this will be
+# ratis_log_service_metadata/ratis_log_service/ratis (unless someone else
registers first, but for many clients, there should only be one).
+#
+# Usually, you would use hadoop-metrics2-phoenix.properties, but we use the
generic
+# hadoop-metrics2.properties to ensure it these are loaded regardless of where
we are running,
+# assuming there isn't another config on the classpath.
+
+#
+# When specifying sinks, the syntax to use is:
+# [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for detail
+
+ratis_test.source.start_mbeans=true
+ratis.sink.sink0.class=org.apache.ratis.metrics.TestHadoop2MetricsSink
+#ratis_test.sink.ganglia.servers=localhost:8649
+#*.sink.graphite.class=org.apache.hadoop.metrics2.sink.GraphiteSink
+#ratis_test.sink.graphite.server_host=localhost
+#ratis_test.sink.graphite.server_port=2003
+# Frequency, in seconds, of sampling from the sources
+*.period=1
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RatisMetrics.java
b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RatisMetrics.java
index 165072a..16cb1d1 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RatisMetrics.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RatisMetrics.java
@@ -19,24 +19,30 @@
package org.apache.ratis.server.metrics;
import java.util.Optional;
+import java.util.concurrent.TimeUnit;
-import com.codahale.metrics.JmxReporter;
import org.apache.ratis.metrics.MetricRegistries;
import org.apache.ratis.metrics.MetricRegistryInfo;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+import org.apache.ratis.metrics.JVMMetrics;
+import org.apache.ratis.metrics.MetricsReporting;
+import org.apache.ratis.metrics.RatisMetricRegistry;
public class RatisMetrics {
- public final static String RATIS_LOG_WORKER_METRICS_DESC = "Ratis Log worker
metrics";
- public final static String RATIS_LOG_WORKER_METRICS_CONTEXT =
"RaftLogWorker";
+ public final static String RATIS_LOG_WORKER_METRICS_DESC = "Ratis metrics";
+ public final static String RATIS_LOG_WORKER_METRICS = "ratis_log_worker";
+ public final static String RATIS_APPLICATION_NAME_METRICS = "ratis_core";
+ static MetricsReporting metricsReporting = new MetricsReporting(500,
TimeUnit.MILLISECONDS);
public static RatisMetricRegistry createMetricRegistryForLogWorker(String
name) {
- return create(new MetricRegistryInfo(name, RATIS_LOG_WORKER_METRICS_DESC,
- RATIS_LOG_WORKER_METRICS_CONTEXT));
+ return create(
+ new MetricRegistryInfo(name, RATIS_APPLICATION_NAME_METRICS,
RATIS_LOG_WORKER_METRICS,
+ RATIS_LOG_WORKER_METRICS_DESC));
}
public static RatisMetricRegistry getMetricRegistryForLogWorker(String name)
{
- return MetricRegistries.global().get(new MetricRegistryInfo(name,
RATIS_LOG_WORKER_METRICS_DESC,
- RATIS_LOG_WORKER_METRICS_CONTEXT)).get();
+ return MetricRegistries.global().get(
+ new MetricRegistryInfo(name, RATIS_APPLICATION_NAME_METRICS,
RATIS_LOG_WORKER_METRICS,
+ RATIS_LOG_WORKER_METRICS_DESC)).get();
}
private static RatisMetricRegistry create(MetricRegistryInfo info) {
@@ -45,7 +51,13 @@ public class RatisMetrics {
return metricRegistry.get();
}
RatisMetricRegistry registry = MetricRegistries.global().create(info);
-
JmxReporter.forRegistry(registry.getDropWizardMetricRegistry()).inDomain("RatisCore").build().start();
+ metricsReporting
+ .startMetricsReporter(registry,
MetricsReporting.MetricReporterType.JMX,
+ MetricsReporting.MetricReporterType.HADOOP2);
+ // JVM metrics
+ JVMMetrics
+ .startJVMReporting(1000, TimeUnit.MILLISECONDS,
MetricsReporting.MetricReporterType.JMX);
+
return registry;
}
}
diff --git
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
index e3d181d..5cfeb89 100644
---
a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
+++
b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java
@@ -22,7 +22,8 @@ import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.metrics.impl.RatisMetricRegistry;
+import org.apache.ratis.metrics.RatisMetricRegistry;
+import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl;
import org.apache.ratis.protocol.RaftPeerId;
import org.apache.ratis.protocol.TimeoutIOException;
import org.apache.ratis.server.RaftServerConfigKeys;
diff --git
a/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java
b/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java
index eb52869..d5c5850 100644
---
a/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java
+++
b/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java
@@ -30,7 +30,8 @@ import java.util.function.Consumer;
public interface RaftStorageTestUtils {
static String getLogFlushTimeMetric(RaftPeerId serverId) {
- return serverId + "." + RatisMetrics.RATIS_LOG_WORKER_METRICS_CONTEXT +
".flush-time";
+ return RatisMetrics.RATIS_APPLICATION_NAME_METRICS + "." +
RatisMetrics.RATIS_LOG_WORKER_METRICS
+ + "." + serverId + ".flush-time";
}
static void printLog(RaftLog log, Consumer<String> println) {
diff --git
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java
index 6660896..971a279 100644
---
a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java
+++
b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java
@@ -111,7 +111,7 @@ public class TestRaftLogMetrics extends BaseTest
Assert.assertTrue(tm.getMeanRate() > 0);
// Test jmx
- ObjectName oname = new ObjectName("RatisCore", "name", flushTimeMetric);
+ ObjectName oname = new ObjectName("ratis_core", "name", flushTimeMetric);
Assert.assertEquals(expectedFlush,
((Long) ManagementFactory.getPlatformMBeanServer().getAttribute(oname,
"Count"))
.intValue());
