tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From spmalle...@apache.org
Subject [14/20] tinkerpop git commit: TINKERPOP-1644 Exposed GremlinScriptEngine metrics in Gremlin Server
Date Wed, 15 Mar 2017 16:36:43 GMT
TINKERPOP-1644 Exposed GremlinScriptEngine metrics in Gremlin Server


Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/b689deb1
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/b689deb1
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/b689deb1

Branch: refs/heads/master
Commit: b689deb1dd1a0c745966a1ed5f8e7e3b2d543af2
Parents: 8ffa5af
Author: Stephen Mallette <spmva@genoprime.com>
Authored: Thu Mar 9 12:25:35 2017 -0500
Committer: Stephen Mallette <spmva@genoprime.com>
Committed: Fri Mar 10 11:13:03 2017 -0500

----------------------------------------------------------------------
 CHANGELOG.asciidoc                              |  2 +-
 .../gremlin/groovy/engine/ScriptEngines.java    |  7 ++
 .../gremlin/server/op/session/Session.java      | 27 +++++++
 .../gremlin/server/util/MetricManager.java      | 79 +++++++++++++++++---
 .../server/util/ServerGremlinExecutor.java      | 15 ++++
 5 files changed, 118 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 3729271..a306dea 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,7 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
 TinkerPop 3.2.5 (Release Date: NOT OFFICIALLY RELEASED YET)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* Added various metrics to the `GremlinGroovyScriptEngine` around script compilation.
+* Added various metrics to the `GremlinGroovyScriptEngine` around script compilation and
exposed them in Gremlin Server.
 * Moved the `caffeine` dependency down to `gremlin-groovy` and out of `gremlin-server`.
 * Improved script compilation in `GremlinGroovyScriptEngine to use better caching, log long
compile times and prevent failed compilations from recompiling on future requests.
 * Script compilation is synchronised.

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
index 7dcfc5c..6dfd87a 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/engine/ScriptEngines.java
@@ -311,6 +311,13 @@ public class ScriptEngines implements AutoCloseable {
     }
 
     /**
+     * Gets an {@link GremlinScriptEngine} by its name.
+     */
+    public GremlinScriptEngine getEngineByName(final String engineName) {
+        return scriptEngines.get(engineName);
+    }
+
+    /**
      * Get the set of {@code ScriptEngine} that implement {@link DependencyManager} interface.
      */
     private Set<DependencyManager> getDependencyManagers() {

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
index 9b16a3b..b8cb28c 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/op/session/Session.java
@@ -18,11 +18,16 @@
  */
 package org.apache.tinkerpop.gremlin.server.op.session;
 
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricFilter;
 import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor;
+import org.apache.tinkerpop.gremlin.groovy.engine.ScriptEngines;
+import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine;
 import org.apache.tinkerpop.gremlin.server.Context;
 import org.apache.tinkerpop.gremlin.server.GraphManager;
 import org.apache.tinkerpop.gremlin.server.Settings;
 import org.apache.tinkerpop.gremlin.server.util.LifeCycleHook;
+import org.apache.tinkerpop.gremlin.server.util.MetricManager;
 import org.apache.tinkerpop.gremlin.server.util.ThreadFactoryUtil;
 import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.slf4j.Logger;
@@ -30,6 +35,7 @@ import org.slf4j.LoggerFactory;
 
 import javax.script.Bindings;
 import javax.script.SimpleBindings;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
@@ -92,6 +98,15 @@ public class Session {
                 SessionOpProcessor.CONFIG_PER_GRAPH_CLOSE_TIMEOUT, SessionOpProcessor.DEFAULT_PER_GRAPH_CLOSE_TIMEOUT).toString());
 
         this.gremlinExecutor = initializeGremlinExecutor().create();
+
+        settings.scriptEngines.keySet().forEach(engineName -> {
+            try {
+                gremlinExecutor.eval("1+1", engineName, Collections.emptyMap()).join();
+                registerMetrics(engineName);
+            } catch (Exception ex) {
+                logger.warn(String.format("Could not initialize {} ScriptEngine as script
could not be evaluated - %s", engineName), ex);
+            }
+        });
     }
 
     public GremlinExecutor getGremlinExecutor() {
@@ -207,6 +222,10 @@ public class Session {
         executor.shutdownNow();
 
         sessions.remove(session);
+
+        // once a session is dead release the gauges in the registry for it
+        MetricManager.INSTANCE.getRegistry().removeMatching((s, metric) -> s.contains(session));
+
         logger.info("Session {} closed", session);
     }
 
@@ -244,4 +263,12 @@ public class Session {
 
         return gremlinExecutorBuilder;
     }
+
+    private void registerMetrics(final String engineName) {
+        final ScriptEngines scriptEngines = gremlinExecutor.getScriptEngines();
+        final GremlinScriptEngine engine = null == scriptEngines ?
+                gremlinExecutor.getScriptEngineManager().getEngineByName(engineName) :
+                scriptEngines.getEngineByName(engineName);
+        MetricManager.INSTANCE.registerGremlinScriptEngineMetrics(engine, "session", session,
"class-cache");
+    }
 }

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java
b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java
index 2867aa7..06179d2 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/MetricManager.java
@@ -33,6 +33,9 @@ import com.codahale.metrics.ganglia.GangliaReporter;
 import com.codahale.metrics.graphite.Graphite;
 import com.codahale.metrics.graphite.GraphiteReporter;
 import info.ganglia.gmetric4j.gmetric.GMetric;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine;
+import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,17 +69,16 @@ public enum MetricManager {
     private GraphiteReporter graphiteReporter = null;
 
     /**
-     * Return the Titan Metrics registry.
+     * Return the {@code MetricsRegistry}.
      *
-     * @return the single {@code MetricRegistry} used for all of Titan's Metrics
-     * monitoring
+     * @return the single {@code MetricRegistry} used for all of monitoring
      */
     public MetricRegistry getRegistry() {
         return registry;
     }
 
     /**
-     * Create a {@link ConsoleReporter} attached to the Titan Metrics registry.
+     * Create a {@link ConsoleReporter} attached to the {@code MetricsRegistry}.
      *
      * @param reportIntervalInMS milliseconds to wait between dumping metrics to the console
      */
@@ -106,7 +108,7 @@ public enum MetricManager {
     }
 
     /**
-     * Create a {@link CsvReporter} attached to the Titan Metrics registry.
+     * Create a {@link CsvReporter} attached to the {@code MetricsRegistry}.
      * <p/>
      * The {@code output} argument must be non-null but need not exist. If it
      * doesn't already exist, this method attempts to create it by calling
@@ -153,7 +155,7 @@ public enum MetricManager {
     }
 
     /**
-     * Create a {@link JmxReporter} attached to the Titan Metrics registry.
+     * Create a {@link JmxReporter} attached to the {@code MetricsRegistry}.
      * <p/>
      * If {@code domain} or {@code agentId} is null, then Metrics's uses its own
      * internal default value(s).
@@ -210,7 +212,7 @@ public enum MetricManager {
     }
 
     /**
-     * Create a {@link Slf4jReporter} attached to the Titan Metrics registry.
+     * Create a {@link Slf4jReporter} attached to the {@code MetricsRegistry}.
      * <p/>
      * If {@code loggerName} is null, or if it is non-null but
      * LoggerFactory.getLogger(loggerName) returns null, then Metrics's
@@ -257,7 +259,7 @@ public enum MetricManager {
     }
 
     /**
-     * Create a {@link GangliaReporter} attached to the Titan Metrics registry.
+     * Create a {@link GangliaReporter} attached to the {@code MetricsRegistry}.
      * <p/>
      * {@code groupOrHost} and {@code addressingMode} must be non-null. The
      * remaining non-primitive arguments may be null. If {@code protocol31} is
@@ -321,7 +323,7 @@ public enum MetricManager {
     }
 
     /**
-     * Create a {@link GraphiteReporter} attached to the Titan Metrics registry.
+     * Create a {@link GraphiteReporter} attached to the {@code MetricsRegistry}.
      * <p/>
      * If {@code prefix} is null, then Metrics's internal default prefix is used
      * (empty string at the time this comment was written).
@@ -367,8 +369,7 @@ public enum MetricManager {
     }
 
     /**
-     * Remove all Titan Metrics reporters previously configured through the
-     * {@code add*} methods on this class.
+     * Remove all reporters previously configured through the {@code add*} methods on this
class.
      */
     public synchronized void removeAllReporters() {
         removeConsoleReporter();
@@ -418,4 +419,60 @@ public enum MetricManager {
     public Histogram getHistogram(final String prefix, final String... names) {
         return getRegistry().histogram(MetricRegistry.name(prefix, names));
     }
+
+    /**
+     * Registers metrics from a {@link GremlinScriptEngine}. At this point, this only works
for the
+     * {@link GremlinGroovyScriptEngine} as it is the only one that collects metrics at this
point. As the
+     * {@link GremlinScriptEngine} implementations achieve greater parity these metrics will
get expanded.
+     */
+    public void registerGremlinScriptEngineMetrics(final GremlinScriptEngine engine, final
String... prefix) {
+        if (engine instanceof GremlinGroovyScriptEngine) {
+            final GremlinGroovyScriptEngine gremlinGroovyScriptEngine = (GremlinGroovyScriptEngine)
engine;
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"long-run-compilation-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLongRunCompilationCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"estimated-size")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEstimatedSize);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"average-load-penalty")),
+                    (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheAverageLoadPenalty);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"eviction-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEvictionCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"eviction-weight")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheEvictionWeight);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"hit-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheHitCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"hit-rate")),
+                    (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheHitRate);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"load-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"load-failure-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadFailureCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"load-failure-rate")),
+                    (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheLoadFailureRate);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"load-success-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheLoadSuccessCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"miss-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheMissCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"miss-rate")),
+                    (Gauge<Double>) gremlinGroovyScriptEngine::getClassCacheMissRate);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"request-count")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheRequestCount);
+            MetricManager.INSTANCE.getRegistry().register(
+                    MetricRegistry.name(GremlinGroovyScriptEngine.class, ArrayUtils.add(prefix,
"total-load-time")),
+                    (Gauge<Long>) gremlinGroovyScriptEngine::getClassCacheTotalLoadTime);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/b689deb1/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java
b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java
index b9b7280..d66e0e8 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/util/ServerGremlinExecutor.java
@@ -18,7 +18,11 @@
  */
 package org.apache.tinkerpop.gremlin.server.util;
 
+import com.codahale.metrics.Gauge;
 import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor;
+import org.apache.tinkerpop.gremlin.groovy.engine.ScriptEngines;
+import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine;
+import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine;
 import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
 import org.apache.tinkerpop.gremlin.server.Channelizer;
 import org.apache.tinkerpop.gremlin.server.GraphManager;
@@ -39,6 +43,8 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
 import java.util.stream.Collectors;
 
+import static com.codahale.metrics.MetricRegistry.name;
+
 /**
  * The core of script execution in Gremlin Server.  Given {@link Settings} and optionally
other arguments, this
  * class will construct a {@link GremlinExecutor} to be used by Gremlin Server.  A typical
usage would be to
@@ -152,6 +158,7 @@ public class ServerGremlinExecutor<T extends ScheduledExecutorService>
{
         settings.scriptEngines.keySet().forEach(engineName -> {
             try {
                 gremlinExecutor.eval("1+1", engineName, Collections.emptyMap()).join();
+                registerMetrics(engineName);
             } catch (Exception ex) {
                 logger.warn(String.format("Could not initialize {} ScriptEngine as script
could not be evaluated - %s", engineName), ex);
             }
@@ -179,6 +186,14 @@ public class ServerGremlinExecutor<T extends ScheduledExecutorService>
{
                 .collect(Collectors.toList());
     }
 
+    private void registerMetrics(final String engineName) {
+        final ScriptEngines scriptEngines = gremlinExecutor.getScriptEngines();
+        final GremlinScriptEngine engine = null == scriptEngines ?
+                gremlinExecutor.getScriptEngineManager().getEngineByName(engineName) :
+                scriptEngines.getEngineByName(engineName);
+        MetricManager.INSTANCE.registerGremlinScriptEngineMetrics(engine, "sessionless",
"class-cache");
+    }
+
     public void addHostOption(final String key, final Object value) {
         hostOptions.put(key, value);
     }


Mime
View raw message