cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jji...@apache.org
Subject cassandra git commit: Add histogram for delay to deliver hints
Date Mon, 13 Mar 2017 04:37:30 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 67e9a5ffd -> 0c5faef66


Add histogram for delay to deliver hints

Patch by Jeff Jirsa; Reviewed by Stefan Podkowinski for CASSANDRA-13234


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0c5faef6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0c5faef6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0c5faef6

Branch: refs/heads/trunk
Commit: 0c5faef664aa403998f59cc77c9b39861890cfa1
Parents: 67e9a5f
Author: Jeff Jirsa <jeff@jeffjirsa.net>
Authored: Fri Feb 17 14:20:49 2017 -0800
Committer: Jeff Jirsa <jeff@jeffjirsa.net>
Committed: Sun Mar 12 21:37:11 2017 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 doc/source/operating/metrics.rst                | 25 ++++++++++++++++
 .../cassandra/hints/EncodedHintMessage.java     |  5 ++++
 src/java/org/apache/cassandra/hints/Hint.java   |  6 ++++
 .../apache/cassandra/hints/HintsDispatcher.java |  3 ++
 .../cassandra/metrics/HintsServiceMetrics.java  | 31 ++++++++++++++++++++
 6 files changed, 71 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3acc2b4..7247f36 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -38,6 +38,7 @@
  * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
  * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
  * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
 
 
 3.11.0

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/doc/source/operating/metrics.rst
----------------------------------------------------------------------
diff --git a/doc/source/operating/metrics.rst b/doc/source/operating/metrics.rst
index 373d4d2..af2e36e 100644
--- a/doc/source/operating/metrics.rst
+++ b/doc/source/operating/metrics.rst
@@ -524,6 +524,31 @@ Hints_created-<PeerIP>       Counter        Number of hints on
disk for this pee
 Hints_not_stored-<PeerIP>    Counter        Number of hints not stored for this peer,
due to being down past the configured hint window.
 =========================== ============== ===========
 
+HintsService Metrics
+^^^^^^^^^^^^^^^^^^^^^
+
+Metrics specific to the Hints delivery service.  There are also some metrics related to hints
tracked in ``Storage Metrics``
+
+These metrics include the peer endpoint **in the metric name**
+
+Reported name format:
+
+**Metric Name**
+    ``org.apache.cassandra.metrics.HintsService.<MetricName>``
+
+**JMX MBean**
+    ``org.apache.cassandra.metrics:type=HintsService name=<MetricName>``
+
+=========================== ============== ===========
+Name                        Type           Description
+=========================== ============== ===========
+HintsSucceeded               Meter          A meter of the hints successfully delivered
+HintsFailed                  Meter          A meter of the hints that failed deliver
+HintsTimedOut                Meter          A meter of the hints that timed out
+Hints_delays                 Histogram      Histogram of hint delivery delays (in milliseconds)
+Hints_delays-<PeerIP>        Histogram      Histogram of hint delivery delays (in milliseconds)
per peer
+=========================== ============== ===========
+
 SSTable Index Metrics
 ^^^^^^^^^^^^^^^^^^^^^
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
index 4fe05ac..50d1302 100644
--- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
+++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java
@@ -58,6 +58,11 @@ final class EncodedHintMessage
         return new MessageOut<>(MessagingService.Verb.HINT, this, serializer);
     }
 
+    public long getHintCreationTime()
+    {
+        return Hint.serializer.getHintCreationTime(hint, version);
+    }
+
     private static class Serializer implements IVersionedSerializer<EncodedHintMessage>
     {
         public long serializedSize(EncodedHintMessage message, int version)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/Hint.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java
index 4e8f139..1582a3c 100644
--- a/src/java/org/apache/cassandra/hints/Hint.java
+++ b/src/java/org/apache/cassandra/hints/Hint.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.hints;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 
@@ -146,5 +147,10 @@ public final class Hint
             int gcgs = (int) in.readUnsignedVInt();
             return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs);
         }
+
+        public long getHintCreationTime(ByteBuffer hintBuffer, int version)
+        {
+            return hintBuffer.getLong(0);
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/HintsDispatcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
index 3ac77a3..6fc55df 100644
--- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java
+++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java
@@ -27,6 +27,7 @@ import java.util.function.Function;
 
 import com.google.common.util.concurrent.RateLimiter;
 
+import org.apache.cassandra.db.monitoring.ApproximateTime;
 import org.apache.cassandra.exceptions.RequestFailureReason;
 import org.apache.cassandra.metrics.HintsServiceMetrics;
 import org.apache.cassandra.net.IAsyncCallbackWithFailure;
@@ -175,6 +176,7 @@ final class HintsDispatcher implements AutoCloseable
     {
         Callback callback = new Callback();
         HintMessage message = new HintMessage(hostId, hint);
+        HintsServiceMetrics.updateDelayMetrics(address , ApproximateTime.currentTimeMillis()
- hint.creationTime);
         MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address,
callback);
         return callback;
     }
@@ -187,6 +189,7 @@ final class HintsDispatcher implements AutoCloseable
     {
         Callback callback = new Callback();
         EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion);
+        HintsServiceMetrics.updateDelayMetrics(address, ApproximateTime.currentTimeMillis()
- message.getHintCreationTime());
         MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address,
callback);
         return callback;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
index ad85281..d6a75f7 100644
--- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java
@@ -17,7 +17,16 @@
  */
 package org.apache.cassandra.metrics;
 
+import java.net.InetAddress;
+
+import com.google.common.util.concurrent.MoreExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 
@@ -26,9 +35,31 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
  */
 public final class HintsServiceMetrics
 {
+    private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class);
+
     private static final MetricNameFactory factory = new DefaultNameFactory("HintsService");
 
     public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded"));
     public static final Meter hintsFailed    = Metrics.meter(factory.createMetricName("HintsFailed"));
     public static final Meter hintsTimedOut  = Metrics.meter(factory.createMetricName("HintsTimedOut"));
+
+    /** Histogram of all hint delivery delays */
+    private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"),
false);
+
+    /** Histograms per-endpoint of hint delivery delays, This is not a cache. */
+    private static final LoadingCache<InetAddress, Histogram> delayByEndpoint = Caffeine.newBuilder()
+        .executor(MoreExecutors.directExecutor())
+        .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':',
'.')), false));
+
+    public static void updateDelayMetrics(InetAddress endpoint, long delay)
+    {
+        if (delay <= 0)
+        {
+            logger.warn("Invalid negative latency in hint delivery delay: {}", delay);
+            return;
+        }
+
+        globalDelayHistogram.update(delay);
+        delayByEndpoint.get(endpoint).update(delay);
+    }
 }


Mime
View raw message