TINKERPOP-1644 Exposed GremlinScriptEngine metrics in Gremlin Server
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/b689deb1 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/b689deb1 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/b689deb1 Branch: refs/heads/TINKERPOP-1644 Commit: b689deb1dd1a0c745966a1ed5f8e7e3b2d543af2 Parents: 8ffa5af Author: Stephen Mallette <sp...@genoprime.com> Authored: Thu Mar 9 12:25:35 2017 -0500 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Fri Mar 10 11:13:03 2017 -0500 ---------------------------------------------------------------------- CHANGELOG.asciidoc | 2 +- .../gremlin/groovy/engine/ScriptEngines.java | 7 ++ .../gremlin/server/op/session/Session.java | 27 +++++++ .../gremlin/server/util/MetricManager.java | 79 +++++++++++++++++--- .../server/util/ServerGremlinExecutor.java | 15 ++++ 5 files changed, 118 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/CHANGELOG.asciidoc ---------------------------------------------------------------------- diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 3729271..a306dea 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,7 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima TinkerPop 3.2.5 (Release Date: NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* Added various metrics to the `GremlinGroovyScriptEngine` around script compilation. +* Added various metrics to the `GremlinGroovyScriptEngine` around script compilation and exposed them in Gremlin Server. * Moved the `caffeine` dependency down to `gremlin-groovy` and out of `gremlin-server`. * Improved script compilation in `GremlinGroovyScriptEngine to use better caching, log long compile times and prevent failed compilations from recompiling on future requests. * Script compilation is synchronised. http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java ---------------------------------------------------------------------- diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java index 7dcfc5c..6dfd87a 100644 --- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java +++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java @@ -311,6 +311,13 @@ public class ScriptEngines implements AutoCloseable { } /** + * Gets an {@link GremlinScriptEngine} by its name. + */ + public GremlinScriptEngine getEngineByName(final String engineName) { + return scriptEngines.get(engineName); + } + + /** * Get the set of {@code ScriptEngine} that implement {@link DependencyManager} interface. */ private Set<DependencyManager> getDependencyManagers() { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java ---------------------------------------------------------------------- diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java index 9b16a3b..b8cb28c 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java @@ -18,11 +18,16 @@ */ package org.apache.tinkerpop.gremlin.server.op.session; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricFilter; import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; +import org.apache.tinkerpop.gremlin.groovy.engine.ScriptEngines; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; import org.apache.tinkerpop.gremlin.server.Context; import org.apache.tinkerpop.gremlin.server.GraphManager; import org.apache.tinkerpop.gremlin.server.Settings; import org.apache.tinkerpop.gremlin.server.util.LifeCycleHook; +import org.apache.tinkerpop.gremlin.server.util.MetricManager; import org.apache.tinkerpop.gremlin.server.util.ThreadFactoryUtil; import org.apache.tinkerpop.gremlin.structure.Graph; import org.slf4j.Logger; @@ -30,6 +35,7 @@ import org.slf4j.LoggerFactory; import javax.script.Bindings; import javax.script.SimpleBindings; +import java.util.Collections; import java.util.HashSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -92,6 +98,15 @@ public class Session { SessionOpProcessor.CONFIG_PER_GRAPH_CLOSE_TIMEOUT, SessionOpProcessor.DEFAULT_PER_GRAPH_CLOSE_TIMEOUT).toString()); this.gremlinExecutor = initializeGremlinExecutor().create(); + + settings.scriptEngines.keySet().forEach(engineName -> { + try { + gremlinExecutor.eval("1+1", engineName, Collections.emptyMap()).join(); + registerMetrics(engineName); + } catch (Exception ex) { + logger.warn(String.format("Could not initialize {} ScriptEngine as script could not be evaluated - %s", engineName), ex); + } + }); } public GremlinExecutor getGremlinExecutor() { @@ -207,6 +222,10 @@ public class Session { executor.shutdownNow(); sessions.remove(session); + + // once a session is dead release the gauges in the registry for it + MetricManager.INSTANCE.getRegistry().removeMatching((s, metric) -> s.contains(session)); + logger.info("Session {} closed", session); } @@ -244,4 +263,12 @@ public class Session { return gremlinExecutorBuilder; } + + private void registerMetrics(final String engineName) { + final ScriptEngines scriptEngines = gremlinExecutor.getScriptEngines(); + final GremlinScriptEngine engine = null == scriptEngines ? + gremlinExecutor.getScriptEngineManager().getEngineByName(engineName) : + scriptEngines.getEngineByName(engineName); + MetricManager.INSTANCE.registerGremlinScriptEngineMetrics(engine, "session", session, "class-cache"); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java ---------------------------------------------------------------------- diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java index 2867aa7..06179d2 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java @@ -33,6 +33,9 @@ import com.codahale.metrics.ganglia.GangliaReporter; import com.codahale.metrics.graphite.Graphite; import com.codahale.metrics.graphite.GraphiteReporter; import info.ganglia.gmetric4j.gmetric.GMetric; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,17 +69,16 @@ public enum MetricManager { private GraphiteReporter graphiteReporter = null; /** - * Return the Titan Metrics registry. + * Return the {@code MetricsRegistry}. * - * @return the single {@code MetricRegistry} used for all of Titan's Metrics - * monitoring + * @return the single {@code MetricRegistry} used for all of monitoring */ public MetricRegistry getRegistry() { return registry; } /** - * Create a {@link ConsoleReporter} attached to the Titan Metrics registry. + * Create a {@link ConsoleReporter} attached to the {@code MetricsRegistry}. * * @param reportIntervalInMS milliseconds to wait between dumping metrics to the console */ @@ -106,7 +108,7 @@ public enum MetricManager { } /** - * Create a {@link CsvReporter} attached to the Titan Metrics registry. + * Create a {@link CsvReporter} attached to the {@code MetricsRegistry}. * <p/> * The {@code output} argument must be non-null but need not exist. If it * doesn't already exist, this method attempts to create it by calling @@ -153,7 +155,7 @@ public enum MetricManager { } /** - * Create a {@link JmxReporter} attached to the Titan Metrics registry. + * Create a {@link JmxReporter} attached to the {@code MetricsRegistry}. * <p/> * If {@code domain} or {@code agentId} is null, then Metrics's uses its own * internal default value(s). @@ -210,7 +212,7 @@ public enum MetricManager { } /** - * Create a {@link Slf4jReporter} attached to the Titan Metrics registry. + * Create a {@link Slf4jReporter} attached to the {@code MetricsRegistry}. * <p/> * If {@code loggerName} is null, or if it is non-null but * LoggerFactory.getLogger(loggerName) returns null, then Metrics's @@ -257,7 +259,7 @@ public enum MetricManager { } /** - * Create a {@link GangliaReporter} attached to the Titan Metrics registry. + * Create a {@link GangliaReporter} attached to the {@code MetricsRegistry}. * <p/> * {@code groupOrHost} and {@code addressingMode} must be non-null. The * remaining non-primitive arguments may be null. If {@code protocol31} is @@ -321,7 +323,7 @@ public enum MetricManager { } /** - * Create a {@link GraphiteReporter} attached to the Titan Metrics registry. + * Create a {@link GraphiteReporter} attached to the {@code MetricsRegistry}. * <p/> * If {@code prefix} is null, then Metrics's internal default prefix is used * (empty string at the time this comment was written). @@ -367,8 +369,7 @@ public enum MetricManager { } /** - * Remove all Titan Metrics reporters previously configured through the - * {@code add*} methods on this class. + * Remove all reporters previously configured through the {@code add*} methods on this class. */ public synchronized void removeAllReporters() { removeConsoleReporter(); @@ -418,4 +419,60 @@ public enum MetricManager { public Histogram getHistogram(final String prefix, final String... names) { return getRegistry().histogram(MetricRegistry.name(prefix, names)); } + + /** + * Registers metrics from a {@link GremlinScriptEngine}. At this point, this only works for the + * {@link GremlinGroovyScriptEngine} as it is the only one that collects metrics at this point. As the + * {@link GremlinScriptEngine} implementations achieve greater parity these metrics will get expanded. + */ + public void registerGremlinScriptEngineMetrics(final GremlinScriptEngine engine, final String... prefix) { + if (engine instanceof GremlinGroovyScriptEngine) { + final GremlinGroovyScriptEngine gremlinGroovyScriptEngine = (GremlinGroovyScriptEngine) engine; + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "long-run-compilation-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLongRunCompilationCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "estimated-size")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEstimatedSize); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "average-load-penalty")), + (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheAverageLoadPenalty); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "eviction-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEvictionCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "eviction-weight")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEvictionWeight); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "hit-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheHitCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "hit-rate")), + (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheHitRate); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "load-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "load-failure-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadFailureCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "load-failure-rate")), + (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheLoadFailureRate); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "load-success-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadSuccessCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "miss-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheMissCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "miss-rate")), + (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheMissRate); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "request-count")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheRequestCount); + MetricManager.INSTANCE.getRegistry().register( + MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix, "total-load-time")), + (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheTotalLoadTime); + } + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java ---------------------------------------------------------------------- diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java index b9b7280..d66e0e8 100644 --- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java +++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java @@ -18,7 +18,11 @@ */ package org.apache.tinkerpop.gremlin.server.util; +import com.codahale.metrics.Gauge; import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; +import org.apache.tinkerpop.gremlin.groovy.engine.ScriptEngines; +import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; import org.apache.tinkerpop.gremlin.server.Channelizer; import org.apache.tinkerpop.gremlin.server.GraphManager; @@ -39,6 +43,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.stream.Collectors; +import static com.codahale.metrics.MetricRegistry.name; + /** * The core of script execution in Gremlin Server. Given {@link Settings} and optionally other arguments, this * class will construct a {@link GremlinExecutor} to be used by Gremlin Server. A typical usage would be to @@ -152,6 +158,7 @@ public class ServerGremlinExecutor<T extends ScheduledExecutorService> { settings.scriptEngines.keySet().forEach(engineName -> { try { gremlinExecutor.eval("1+1", engineName, Collections.emptyMap()).join(); + registerMetrics(engineName); } catch (Exception ex) { logger.warn(String.format("Could not initialize {} ScriptEngine as script could not be evaluated - %s", engineName), ex); } @@ -179,6 +186,14 @@ public class ServerGremlinExecutor<T extends ScheduledExecutorService> { .collect(Collectors.toList()); } + private void registerMetrics(final String engineName) { + final ScriptEngines scriptEngines = gremlinExecutor.getScriptEngines(); + final GremlinScriptEngine engine = null == scriptEngines ? + gremlinExecutor.getScriptEngineManager().getEngineByName(engineName) : + scriptEngines.getEngineByName(engineName); + MetricManager.INSTANCE.registerGremlinScriptEngineMetrics(engine, "sessionless", "class-cache"); + } + public void addHostOption(final String key, final Object value) { hostOptions.put(key, value); }
