cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brandonwilli...@apache.org
Subject [10/16] git commit: Expose global CF metrics
Date Tue, 24 Jun 2014 18:09:07 GMT
Expose global CF metrics

Patch by Chris Lohfink, reviewed by brandonwilliams for CASSANDRA-7273


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/94ff6394
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/94ff6394
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/94ff6394

Branch: refs/heads/cassandra-2.1
Commit: 94ff639429a65acb5f122ed559e98dd60a40e42d
Parents: f56ba7a
Author: Brandon Williams <brandonwilliams@apache.org>
Authored: Tue Jun 24 13:05:27 2014 -0500
Committer: Brandon Williams <brandonwilliams@apache.org>
Committed: Tue Jun 24 13:05:27 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |   4 +-
 src/java/org/apache/cassandra/db/Keyspace.java  |   2 +-
 .../cassandra/metrics/ColumnFamilyMetrics.java  | 325 +++++++++++++++----
 .../cassandra/metrics/KeyspaceMetrics.java      | 180 +++++-----
 .../cassandra/metrics/LatencyMetrics.java       |  28 +-
 .../org/apache/cassandra/db/KeyspaceTest.java   |  16 +-
 7 files changed, 411 insertions(+), 145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 530d70d..d23ce37 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,7 @@
  * Improve sub range repair validation (CASSANDRA-7317)
  * Accept subtypes for function results, type casts (CASSANDRA-6766)
 Merged from 1.2:
+ * Expose global ColumnFamily metrics (CASSANDRA-7273)
  * Handle possible integer overflow in FastByteArrayOutputStream (CASSANDRA-7373)
  * cqlsh: 'ascii' values weren't formatted as text (CASSANDRA-7407)
  * cqlsh: ignore .cassandra permission errors (CASSANDRA-7266)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c641f3a..2824924 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2266,12 +2266,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
     public double getTombstonesPerSlice()
     {
-        return metric.tombstoneScannedHistogram.getSnapshot().getMedian();
+        return metric.tombstoneScannedHistogram.cf.getSnapshot().getMedian();
     }
 
     public double getLiveCellsPerSlice()
     {
-        return metric.liveScannedHistogram.getSnapshot().getMedian();
+        return metric.liveScannedHistogram.cf.getSnapshot().getMedian();
     }
 
     // End JMX get/set.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/src/java/org/apache/cassandra/db/Keyspace.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index a6da637..28045f4 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -265,12 +265,12 @@ public class Keyspace
         assert metadata != null : "Unknown keyspace " + keyspaceName;
         createReplicationStrategy(metadata);
 
+        this.metric = new KeyspaceMetrics(this);
         for (CFMetaData cfm : new ArrayList<CFMetaData>(metadata.cfMetaData().values()))
         {
             logger.debug("Initializing {}.{}", getName(), cfm.cfName);
             initCf(cfm.cfId, cfm.cfName, loadSSTables);
         }
-        this.metric = new KeyspaceMetrics(this);
     }
 
     public void createReplicationStrategy(KSMetaData ksm)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index 50c80fc..3dc269f 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -17,18 +17,23 @@
  */
 package org.apache.cassandra.metrics;
 
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.*;
-import com.yammer.metrics.util.RatioGauge;
-
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.io.sstable.SSTableMetadata;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.utils.EstimatedHistogram;
 
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.*;
+import com.yammer.metrics.util.RatioGauge;
+
 /**
  * Metrics for {@link ColumnFamilyStore}.
  */
@@ -49,7 +54,7 @@ public class ColumnFamilyMetrics
     /** Histogram of estimated number of columns. */
     public final Gauge<long[]> estimatedColumnCountHistogram;
     /** Histogram of the number of sstable data files accessed per read */
-    public final Histogram sstablesPerReadHistogram;
+    public final ColumnFamilyHistogram sstablesPerReadHistogram;
     /** (Local) read metrics */
     public final LatencyMetrics readLatency;
     /** (Local) range slice metrics */
@@ -85,21 +90,36 @@ public class ColumnFamilyMetrics
     /** Key cache hit rate  for this CF */
     public final Gauge<Double> keyCacheHitRate;
     /** Tombstones scanned in queries on this CF */
-    public final Histogram tombstoneScannedHistogram;
+    public final ColumnFamilyHistogram tombstoneScannedHistogram;
     /** Live cells scanned in queries on this CF */
-    public final Histogram liveScannedHistogram;
+    public final ColumnFamilyHistogram liveScannedHistogram;
 
     public final Timer coordinatorReadLatency;
     public final Timer coordinatorScanLatency;
 
     private final MetricNameFactory factory;
+    private static final MetricNameFactory globalNameFactory = new AllColumnFamilyMetricNameFactory();;
 
     public final Counter speculativeRetries;
 
     // for backward compatibility
     @Deprecated public final EstimatedHistogram sstablesPerRead = new EstimatedHistogram(35);
     @Deprecated public final EstimatedHistogram recentSSTablesPerRead = new EstimatedHistogram(35);
-
+    
+    public final static LatencyMetrics globalReadLatency = new LatencyMetrics(globalNameFactory,
"Read");
+    public final static LatencyMetrics globalWriteLatency = new LatencyMetrics(globalNameFactory,
"Write");
+    public final static LatencyMetrics globalRangeLatency = new LatencyMetrics(globalNameFactory,
"Range");
+    
+    /**
+     * stores metrics that will be rolled into a single global metric
+     */
+    public final static ConcurrentMap<String, Set<Metric>> allColumnFamilyMetrics
= Maps.newConcurrentMap();
+    
+    /**
+     * Stores all metric names created that can be used when unregistering
+     */
+    public final static Set<String> all = Sets.newHashSet();
+    
     /**
      * Creates metrics for given {@link ColumnFamilyStore}.
      *
@@ -109,28 +129,28 @@ public class ColumnFamilyMetrics
     {
         factory = new ColumnFamilyMetricNameFactory(cfs);
 
-        memtableColumnsCount = Metrics.newGauge(factory.createMetricName("MemtableColumnsCount"),
new Gauge<Long>()
+        memtableColumnsCount = createColumnFamilyGauge("MemtableColumnsCount", new Gauge<Long>()
         {
             public Long value()
             {
                 return cfs.getDataTracker().getMemtable().getOperations();
             }
         });
-        memtableDataSize = Metrics.newGauge(factory.createMetricName("MemtableDataSize"),
new Gauge<Long>()
+        memtableDataSize = createColumnFamilyGauge("MemtableDataSize", new Gauge<Long>()
         {
             public Long value()
             {
                 return cfs.getDataTracker().getMemtable().getLiveSize();
             }
         });
-        allMemtablesDataSize = Metrics.newGauge(factory.createMetricName("AllMemtablesDataSize"),
new Gauge<Long>()
+        allMemtablesDataSize = createColumnFamilyGauge("AllMemtablesDataSize", new Gauge<Long>()
         {
             public Long value()
             {
                 return cfs.getTotalAllMemtablesLiveSize();
             }
         });
-        memtableSwitchCount = Metrics.newCounter(factory.createMetricName("MemtableSwitchCount"));
+        memtableSwitchCount = createColumnFamilyCounter("MemtableSwitchCount");
         estimatedRowSizeHistogram = Metrics.newGauge(factory.createMetricName("EstimatedRowSizeHistogram"),
new Gauge<long[]>()
         {
             public long[] value()
@@ -159,8 +179,8 @@ public class ColumnFamilyMetrics
                 return histogram;
             }
         });
-        sstablesPerReadHistogram = Metrics.newHistogram(factory.createMetricName("SSTablesPerReadHistogram"),
true);
-        compressionRatio = Metrics.newGauge(factory.createMetricName("CompressionRatio"),
new Gauge<Double>()
+        sstablesPerReadHistogram = createColumnFamilyHistogram("SSTablesPerReadHistogram",
cfs.keyspace.metric.sstablesPerReadHistogram);
+        compressionRatio = createColumnFamilyGauge("CompressionRatio", new Gauge<Double>()
         {
             public Double value()
             {
@@ -174,13 +194,32 @@ public class ColumnFamilyMetrics
                         total++;
                     }
                 }
-                return total != 0 ? (double) sum / total : 0;
+                return total != 0 ? sum / total : 0;
+            }
+        }, new Gauge<Double>() // global gauge
+        {
+            public Double value()
+            {
+                double sum = 0;
+                int total = 0;
+                for (Keyspace keyspace : Keyspace.all())
+                {
+                    for (SSTableReader sstable : keyspace.getAllSSTables())
+                    {
+                        if (sstable.getCompressionRatio() != SSTableMetadata.NO_COMPRESSION_RATIO)
+                        {
+                            sum += sstable.getCompressionRatio();
+                            total++;
+                        }
+                    }
+                }
+                return total != 0 ? sum / total : 0;
             }
         });
-        readLatency = new LatencyMetrics(factory, "Read");
-        writeLatency = new LatencyMetrics(factory, "Write");
-        rangeLatency = new LatencyMetrics(factory, "Range");
-        pendingCompactions = Metrics.newGauge(factory.createMetricName("PendingCompactions"),
new Gauge<Integer>()
+        readLatency = new LatencyMetrics(factory, "Read", cfs.keyspace.metric.readLatency,
globalReadLatency);
+        writeLatency = new LatencyMetrics(factory, "Write", cfs.keyspace.metric.writeLatency,
globalWriteLatency);
+        rangeLatency = new LatencyMetrics(factory, "Range", cfs.keyspace.metric.rangeLatency,
globalRangeLatency);
+        pendingCompactions = createColumnFamilyGauge("PendingCompactions", new Gauge<Integer>()
         {
             public Integer value()
             {
@@ -195,16 +234,16 @@ public class ColumnFamilyMetrics
                 return Keyspace.switchLock.getQueueLength();
             }
         });
-        liveSSTableCount = Metrics.newGauge(factory.createMetricName("LiveSSTableCount"),
new Gauge<Integer>()
+        liveSSTableCount = createColumnFamilyGauge("LiveSSTableCount", new Gauge<Integer>()
         {
             public Integer value()
             {
                 return cfs.getDataTracker().getSSTables().size();
             }
         });
-        liveDiskSpaceUsed = Metrics.newCounter(factory.createMetricName("LiveDiskSpaceUsed"));
-        totalDiskSpaceUsed = Metrics.newCounter(factory.createMetricName("TotalDiskSpaceUsed"));
-        minRowSize = Metrics.newGauge(factory.createMetricName("MinRowSize"), new Gauge<Long>()
+        liveDiskSpaceUsed = createColumnFamilyCounter("LiveDiskSpaceUsed");
+        totalDiskSpaceUsed = createColumnFamilyCounter("TotalDiskSpaceUsed");
+        minRowSize = createColumnFamilyGauge("MinRowSize", new Gauge<Long>()
         {
             public Long value()
             {
@@ -216,8 +255,19 @@ public class ColumnFamilyMetrics
                 }
                 return min;
             }
+        }, new Gauge<Long>() // global gauge
+        {
+            public Long value()
+            {
+                long min = Long.MAX_VALUE;
+                for (Metric cfGauge : allColumnFamilyMetrics.get("MinRowSize"))
+                {
+                    min = Math.min(min, ((Gauge<? extends Number>) cfGauge).value().longValue());
+                }
+                return min;
+            }
         });
-        maxRowSize = Metrics.newGauge(factory.createMetricName("MaxRowSize"), new Gauge<Long>()
+        maxRowSize = createColumnFamilyGauge("MaxRowSize", new Gauge<Long>()
         {
             public Long value()
             {
@@ -229,8 +279,19 @@ public class ColumnFamilyMetrics
                 }
                 return max;
             }
+        }, new Gauge<Long>() // global gauge
+        {
+            public Long value()
+            {
+                long max = 0;
+                for (Metric cfGauge : allColumnFamilyMetrics.get("MaxRowSize"))
+                {
+                    max = Math.max(max, ((Gauge<? extends Number>) cfGauge).value().longValue());
+                }
+                return max;
+            }
         });
-        meanRowSize = Metrics.newGauge(factory.createMetricName("MeanRowSize"), new Gauge<Long>()
+        meanRowSize = createColumnFamilyGauge("MeanRowSize", new Gauge<Long>()
         {
             public Long value()
             {
@@ -244,8 +305,25 @@ public class ColumnFamilyMetrics
                 }
                 return count > 0 ? sum / count : 0;
             }
+        }, new Gauge<Long>() // global gauge
+        {
+            public Long value()
+            {
+                long sum = 0;
+                long count = 0;
+                for (Keyspace keyspace : Keyspace.all())
+                {
+                    for (SSTableReader sstable : keyspace.getAllSSTables())
+                    {
+                        long n = sstable.getEstimatedRowSize().count();
+                        sum += sstable.getEstimatedRowSize().mean() * n;
+                        count += n;
+                    }
+                }
+                return count > 0 ? sum / count : 0;
+            }
         });
-        bloomFilterFalsePositives = Metrics.newGauge(factory.createMetricName("BloomFilterFalsePositives"),
new Gauge<Long>()
+        bloomFilterFalsePositives = createColumnFamilyGauge("BloomFilterFalsePositives",
new Gauge<Long>()
         {
             public Long value()
             {
@@ -255,23 +333,23 @@ public class ColumnFamilyMetrics
                 return count;
             }
         });
-        recentBloomFilterFalsePositives = Metrics.newGauge(factory.createMetricName("RecentBloomFilterFalsePositives"),
new Gauge<Long>()
+        recentBloomFilterFalsePositives = createColumnFamilyGauge("RecentBloomFilterFalsePositives",
new Gauge<Long>()
         {
             public Long value()
             {
                 long count = 0L;
-                for (SSTableReader sstable: cfs.getSSTables())
+                for (SSTableReader sstable : cfs.getSSTables())
                     count += sstable.getRecentBloomFilterFalsePositiveCount();
                 return count;
             }
         });
-        bloomFilterFalseRatio = Metrics.newGauge(factory.createMetricName("BloomFilterFalseRatio"),
new Gauge<Double>()
+        bloomFilterFalseRatio = createColumnFamilyGauge("BloomFilterFalseRatio", new Gauge<Double>()
         {
             public Double value()
             {
                 long falseCount = 0L;
                 long trueCount = 0L;
-                for (SSTableReader sstable: cfs.getSSTables())
+                for (SSTableReader sstable : cfs.getSSTables())
                 {
                     falseCount += sstable.getBloomFilterFalsePositiveCount();
                     trueCount += sstable.getBloomFilterTruePositiveCount();
@@ -280,8 +358,26 @@ public class ColumnFamilyMetrics
                     return 0d;
                 return (double) falseCount / (trueCount + falseCount);
             }
+        }, new Gauge<Double>() // global gauge
+        {
+            public Double value()
+            {
+                long falseCount = 0L;
+                long trueCount = 0L;
+                for (Keyspace keyspace : Keyspace.all())
+                {
+                    for (SSTableReader sstable : keyspace.getAllSSTables())
+                    {
+                        falseCount += sstable.getBloomFilterFalsePositiveCount();
+                        trueCount += sstable.getBloomFilterTruePositiveCount();
+                    }
+                }
+                if (falseCount == 0L && trueCount == 0L)
+                    return 0d;
+                return (double) falseCount / (trueCount + falseCount);
+            }
         });
-        recentBloomFilterFalseRatio = Metrics.newGauge(factory.createMetricName("RecentBloomFilterFalseRatio"),
new Gauge<Double>()
+        recentBloomFilterFalseRatio = createColumnFamilyGauge("RecentBloomFilterFalseRatio",
new Gauge<Double>()
         {
             public Double value()
             {
@@ -296,8 +392,26 @@ public class ColumnFamilyMetrics
                     return 0d;
                 return (double) falseCount / (trueCount + falseCount);
             }
+        }, new Gauge<Double>() // global gauge
+        {
+            public Double value()
+            {
+                long falseCount = 0L;
+                long trueCount = 0L;
+                for (Keyspace keyspace : Keyspace.all())
+                {
+                    for (SSTableReader sstable : keyspace.getAllSSTables())
+                    {
+                        falseCount += sstable.getRecentBloomFilterFalsePositiveCount();
+                        trueCount += sstable.getRecentBloomFilterTruePositiveCount();
+                    }
+                }
+                if (falseCount == 0L && trueCount == 0L)
+                    return 0d;
+                return (double) falseCount / (trueCount + falseCount);
+            }
         });
-        bloomFilterDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("BloomFilterDiskSpaceUsed"),
new Gauge<Long>()
+        bloomFilterDiskSpaceUsed = createColumnFamilyGauge("BloomFilterDiskSpaceUsed", new
Gauge<Long>()
         {
             public Long value()
             {
@@ -307,7 +421,7 @@ public class ColumnFamilyMetrics
                 return total;
             }
         });
-        speculativeRetries = Metrics.newCounter(factory.createMetricName("SpeculativeRetries"));
+        speculativeRetries = createColumnFamilyCounter("SpeculativeRetries");
         keyCacheHitRate = Metrics.newGauge(factory.createMetricName("KeyCacheHitRate"), new
RatioGauge()
         {
             protected double getNumerator()
@@ -326,8 +440,8 @@ public class ColumnFamilyMetrics
                 return Math.max(requests, 1); // to avoid NaN.
             }
         });
-        tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"),
true);
-        liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"),
true);
+        tombstoneScannedHistogram = createColumnFamilyHistogram("TombstoneScannedHistogram",
cfs.keyspace.metric.tombstoneScannedHistogram);
+        liveScannedHistogram = createColumnFamilyHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram);
         coordinatorReadLatency = Metrics.newTimer(factory.createMetricName("CoordinatorReadLatency"),
TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
         coordinatorScanLatency = Metrics.newTimer(factory.createMetricName("CoordinatorScanLatency"),
TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
     }
@@ -344,37 +458,123 @@ public class ColumnFamilyMetrics
      */
     public void release()
     {
+        for(String name : all)
+        {
+            allColumnFamilyMetrics.get(name).remove(Metrics.defaultRegistry().allMetrics().get(factory.createMetricName(name)));
+            Metrics.defaultRegistry().removeMetric(factory.createMetricName(name));
+        }
         readLatency.release();
         writeLatency.release();
         rangeLatency.release();
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("AllMemtablesDataSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableColumnsCount"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableDataSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableSwitchCount"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("CompressionRatio"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedRowSizeHistogram"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedColumnCountHistogram"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("SSTablesPerReadHistogram"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("PendingTasks"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveSSTableCount"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveDiskSpaceUsed"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("TotalDiskSpaceUsed"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MinRowSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MaxRowSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MeanRowSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterFalsePositives"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalsePositives"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterFalseRatio"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("SpeculativeRetries"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorReadLatency"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorScanLatency"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("PendingTasks"));
     }
 
+    /**
+     * Create a gauge that will be part of a merged version of all column families.  The
global gauge
+     * will merge each CF gauge by adding their values 
+     */
+    protected <T extends Number> Gauge<T> createColumnFamilyGauge(final String
name, Gauge<T> gauge)
+    {
+        return createColumnFamilyGauge(name, gauge, new Gauge<Long>()
+        {
+            public Long value()
+            {
+                long total = 0;
+                for (Metric cfGauge : allColumnFamilyMetrics.get(name))
+                {
+                    total = total + ((Gauge<? extends Number>) cfGauge).value().longValue();
+                }
+                return total;
+            }
+        });
+    }
+    
+    /**
+     * Create a gauge that will be part of a merged version of all column families.  The
global gauge
+     * is defined as the globalGauge parameter
+     */
+    protected <G,T> Gauge<T> createColumnFamilyGauge(String name, Gauge<T>
gauge, Gauge<G> globalGauge)
+    {
+        Gauge<T> cfGauge = Metrics.newGauge(factory.createMetricName(name), gauge);
+        if (register(name, cfGauge))
+        {
+            Metrics.newGauge(globalNameFactory.createMetricName(name), globalGauge);
+        }
+        return cfGauge;
+    }
+    
+    /**
+     * Creates a counter that will also have a global counter thats the sum of all counters
across 
+     * different column families
+     */
+    protected Counter createColumnFamilyCounter(final String name)
+    {
+        Counter cfCounter = Metrics.newCounter(factory.createMetricName(name));
+        if (register(name, cfCounter))
+        {
+            Metrics.newGauge(globalNameFactory.createMetricName(name), new Gauge<Long>()
+            {
+                public Long value()
+                {
+                    long total = 0;
+                    for (Metric cfGauge : allColumnFamilyMetrics.get(name))
+                    {
+                        total += ((Counter) cfGauge).count();
+                    }
+                    return total;
+                }
+            });
+        }
+        return cfCounter;
+    }
+    
+    /**
+     * Create a histogram-like interface that will register both a CF, keyspace and global
level
+     * histogram and forward any updates to both
+     */
+    protected ColumnFamilyHistogram createColumnFamilyHistogram(String name, Histogram keyspaceHistogram)

+    {
+        Histogram cfHistogram = Metrics.newHistogram(factory.createMetricName(name), true);
 
+        register(name, cfHistogram);
+        return new ColumnFamilyHistogram(cfHistogram, keyspaceHistogram, Metrics.newHistogram(globalNameFactory.createMetricName(name),
true));
+    }
+    
+    /**
+     * Registers a metric to be removed when unloading CF.
+     * @return true if first time metric with that name has been registered
+     */
+    private boolean register(String name, Metric metric)
+    { 
+        boolean ret = allColumnFamilyMetrics.putIfAbsent(name,  new HashSet<Metric>())
== null;
+        allColumnFamilyMetrics.get(name).add(metric);
+        all.add(name);
+        return ret;
+    }
+    
+    public class ColumnFamilyHistogram
+    {
+        public final Histogram[] all;
+        public final Histogram cf;
+        private ColumnFamilyHistogram(Histogram cf, Histogram keyspace, Histogram global)

+        {
+            this.cf = cf;
+            this.all = new Histogram[]{cf, keyspace, global};
+        }
+        
+        public void update(long i) 
+        {
+            for(Histogram histo : all)
+            {
+                histo.update(i);
+            }
+        }
+    }
+    
     class ColumnFamilyMetricNameFactory implements MetricNameFactory
     {
         private final String keyspaceName;
@@ -403,4 +603,17 @@ public class ColumnFamilyMetrics
             return new MetricName(groupName, type, metricName, keyspaceName + "." + columnFamilyName,
mbeanName.toString());
         }
     }
+    
+    static class AllColumnFamilyMetricNameFactory implements MetricNameFactory
+    {
+        public MetricName createMetricName(String metricName)
+        {
+            String groupName = ColumnFamilyMetrics.class.getPackage().getName(); 
+            StringBuilder mbeanName = new StringBuilder();
+            mbeanName.append(groupName).append(":");
+            mbeanName.append("type=ColumnFamily");
+            mbeanName.append(",name=").append(metricName);
+            return new MetricName(groupName, "ColumnFamily", metricName, "all", mbeanName.toString());
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index 4a0980f..7dcf9d5 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -17,16 +17,16 @@
  */
 package org.apache.cassandra.metrics;
 
-import java.util.ArrayList;
-import java.util.List;
-
-import com.yammer.metrics.Metrics;
-import com.yammer.metrics.core.*;
-import com.yammer.metrics.stats.Snapshot;
+import java.util.Set;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.*;
+
 /**
  * Metrics for {@link ColumnFamilyStore}.
  */
@@ -43,16 +43,32 @@ public class KeyspaceMetrics
     /** Estimated number of tasks pending for this column family */
     public final Gauge<Integer> pendingTasks;
     /** Estimate of number of pending compactios for this CF */
-    public final Gauge<Integer> pendingCompactions;
+    public final Gauge<Long> pendingCompactions;
     /** Disk space used by SSTables belonging to this CF */
     public final Gauge<Long> liveDiskSpaceUsed;
     /** Total disk space used by SSTables belonging to this CF, including obsolete ones waiting
to be GC'd */
     public final Gauge<Long> totalDiskSpaceUsed;
     /** Disk space used by bloom filter */
     public final Gauge<Long> bloomFilterDiskSpaceUsed;
+    /** (Local) read metrics */
+    public final LatencyMetrics readLatency;
+    /** (Local) range slice metrics */
+    public final LatencyMetrics rangeLatency;
+    /** (Local) write metrics */
+    public final LatencyMetrics writeLatency;
+    /** Histogram of the number of sstable data files accessed per read */
+    public final Histogram sstablesPerReadHistogram;
+    /** Tombstones scanned in queries on this Keyspace */
+    public final Histogram tombstoneScannedHistogram;
+    /** Live cells scanned in queries on this Keyspace */
+    public final Histogram liveScannedHistogram;
 
     private final MetricNameFactory factory;
-
+    private Keyspace keyspace;
+    
+    /** set containing names of all the metrics stored here, for releasing later */
+    private Set<String> allMetrics = Sets.newHashSet();
+    
     /**
      * Creates metrics for given {@link ColumnFamilyStore}.
      *
@@ -61,63 +77,40 @@ public class KeyspaceMetrics
     public KeyspaceMetrics(final Keyspace ks)
     {
         factory = new KeyspaceMetricNameFactory(ks);
-
-        memtableColumnsCount = Metrics.newGauge(factory.createMetricName("MemtableColumnsCount"),
new Gauge<Long>()
+        keyspace = ks;
+        memtableColumnsCount = createKeyspaceGauge("MemtableColumnsCount", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long total = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    total += cf.metric.memtableColumnsCount.value();
-                }
-                return total;
+                return metric.memtableColumnsCount.value();
             }
         });
-        memtableDataSize = Metrics.newGauge(factory.createMetricName("MemtableDataSize"),
new Gauge<Long>()
+        memtableDataSize = createKeyspaceGauge("MemtableDataSize", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long total = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    total += cf.metric.memtableDataSize.value();
-                }
-                return total;
+                return metric.memtableDataSize.value();
             }
-        });
-        allMemtablesDataSize = Metrics.newGauge(factory.createMetricName("AllMemtablesDataSize"),
new Gauge<Long>()
+        }); 
+        allMemtablesDataSize = createKeyspaceGauge("AllMemtablesDataSize", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long total = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    total += cf.metric.allMemtablesDataSize.value();
-                }
-                return total;
+                return metric.allMemtablesDataSize.value();
             }
         });
-        memtableSwitchCount = Metrics.newGauge(factory.createMetricName("MemtableSwitchCount"),
new Gauge<Long>()
+        memtableSwitchCount = createKeyspaceGauge("MemtableSwitchCount", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long sum = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                    sum += cf.metric.memtableSwitchCount.count();
-                return sum;
+                return metric.memtableSwitchCount.count();
             }
         });
-        pendingCompactions = Metrics.newGauge(factory.createMetricName("PendingCompactions"),
new Gauge<Integer>()
+        pendingCompactions = createKeyspaceGauge("PendingCompactions", new MetricValue()
         {
-            public Integer value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                int sum = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    sum += cf.metric.pendingCompactions.value();
-                }
-                return sum;
+                return (long) metric.pendingCompactions.value();
             }
         });
         pendingTasks = Metrics.newGauge(factory.createMetricName("PendingTasks"), new Gauge<Integer>()
@@ -127,40 +120,37 @@ public class KeyspaceMetrics
                 return Keyspace.switchLock.getQueueLength();
             }
         });
-        liveDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("LiveDiskSpaceUsed"),
new Gauge<Long>()
+        liveDiskSpaceUsed = createKeyspaceGauge("LiveDiskSpaceUsed", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long sum = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    sum += cf.metric.liveDiskSpaceUsed.count();
-                }
-                return sum;
+                return metric.liveDiskSpaceUsed.count();
             }
         });
-        totalDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("TotalDiskSpaceUsed"),
new Gauge<Long>()
+        totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long sum = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                {
-                    sum += cf.metric.totalDiskSpaceUsed.count();
-                }
-                return sum;
+                return metric.totalDiskSpaceUsed.count();
             }
         });
-        bloomFilterDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("BloomFilterDiskSpaceUsed"),
new Gauge<Long>()
+        bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed", new MetricValue()
         {
-            public Long value()
+            public Long getValue(ColumnFamilyMetrics metric)
             {
-                long total = 0;
-                for (ColumnFamilyStore cf : ks.getColumnFamilyStores())
-                    total += cf.metric.bloomFilterDiskSpaceUsed.value();
-                return total;
+                return metric.bloomFilterDiskSpaceUsed.value();
             }
         });
+        // latency metrics for ColumnFamilyMetrics to update
+        readLatency = new LatencyMetrics(factory, "Read");
+        writeLatency = new LatencyMetrics(factory, "Write");
+        rangeLatency = new LatencyMetrics(factory, "Range");
+        // create histograms for ColumnFamilyMetrics to replicate updates to
+        sstablesPerReadHistogram = Metrics.newHistogram(factory.createMetricName("SSTablesPerReadHistogram"),
true);
+        tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"),
true);
+        liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"),
true);
+        // add manually since histograms do not use createKeyspaceGauge method
+        allMetrics.addAll(Lists.newArrayList("SSTablesPerReadHistogram", "TombstoneScannedHistogram",
"LiveScannedHistogram"));
     }
 
     /**
@@ -168,13 +158,51 @@ public class KeyspaceMetrics
      */
     public void release()
     {
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("AllMemtablesDataSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableDataSize"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableSwitchCount"));
+        for(String name : allMetrics) 
+        {
+            Metrics.defaultRegistry().removeMetric(factory.createMetricName(name));
+        }
+        // latency metrics contain multiple metrics internally and need to be released manually
+        readLatency.release();
+        writeLatency.release();
+        rangeLatency.release();
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("PendingTasks"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveDiskSpaceUsed"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("TotalDiskSpaceUsed"));
-        Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
+    }
+    
+    /**
+     * Represents a column family metric value.
+     */
+    private interface MetricValue
+    {
+        /**
+         * get value of a metric
+         * @param columnfamilymetrics of a column family in this keyspace
+         * @return current value of a metric
+         */
+        public Long getValue(ColumnFamilyMetrics metric);
+    }
+
+    /**
+     * Creates a gauge that will sum the current value of a metric for all column families
in this keyspace
+     * @param name
+     * @param MetricValue 
+     * @return Gauge&gt;Long> that computes sum of MetricValue.getValue()
+     */
+    private <T extends Number> Gauge<Long> createKeyspaceGauge(String name, final
MetricValue extractor)
+    {
+        allMetrics.add(name);
+        return Metrics.newGauge(factory.createMetricName(name), new Gauge<Long>()
+        {
+            public Long value()
+            {
+                long sum = 0;
+                for (ColumnFamilyStore cf : keyspace.getColumnFamilyStores())
+                {
+                    sum += extractor.getValue(cf.metric);
+                }
+                return sum;
+            }
+        });
     }
 
     class KeyspaceMetricNameFactory implements MetricNameFactory

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
index b456e2a..28256fe 100644
--- a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java
@@ -17,14 +17,17 @@
  */
 package org.apache.cassandra.metrics;
 
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.cassandra.utils.EstimatedHistogram;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import com.yammer.metrics.Metrics;
 import com.yammer.metrics.core.Counter;
 import com.yammer.metrics.core.Timer;
 
-import org.apache.cassandra.utils.EstimatedHistogram;
-
 /**
  * Metrics about latencies
  */
@@ -35,6 +38,9 @@ public class LatencyMetrics
     /** Total latency in micro sec */
     public final Counter totalLatency;
 
+    /** parent metrics to replicate any updates to **/
+    private List<LatencyMetrics> parents = Lists.newArrayList();
+    
     protected final MetricNameFactory factory;
     protected final String namePrefix;
 
@@ -80,6 +86,20 @@ public class LatencyMetrics
         latency = Metrics.newTimer(factory.createMetricName(namePrefix + "Latency"), TimeUnit.MICROSECONDS,
TimeUnit.SECONDS);
         totalLatency = Metrics.newCounter(factory.createMetricName(namePrefix + "TotalLatency"));
     }
+    
+    /**
+     * Create LatencyMetrics with given group, type, prefix to append to each metric name,
and scope.  Any updates
+     * to this will also run on parent
+     *
+     * @param factory MetricName factory to use
+     * @param namePrefix Prefix to append to each metric name
+     * @param parents... any amount of parents to replicate updates to
+     */
+    public LatencyMetrics(MetricNameFactory factory, String namePrefix, LatencyMetrics ...
parents)
+    {
+        this(factory, namePrefix);
+        this.parents.addAll(ImmutableList.copyOf(parents));
+    }
 
     /** takes nanoseconds **/
     public void addNano(long nanos)
@@ -89,6 +109,10 @@ public class LatencyMetrics
         totalLatency.inc(nanos / 1000);
         totalLatencyHistogram.add(nanos / 1000);
         recentLatencyHistogram.add(nanos / 1000);
+        for(LatencyMetrics parent : parents)
+        {
+            parent.addNano(nanos);
+        }
     }
 
     public void release()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/94ff6394/test/unit/org/apache/cassandra/db/KeyspaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index e04a9d4..efa43cf 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -465,18 +465,18 @@ public class KeyspaceTest extends SchemaLoader
             rm.apply();
             cfStore.forceBlockingFlush();
         }
-        cfStore.metric.sstablesPerReadHistogram.clear();
+        cfStore.metric.sstablesPerReadHistogram.cf.clear();
         ColumnFamily cf = cfStore.getColumnFamily(key, ByteBufferUtil.bytes(""), ByteBufferUtil.bytes("col1499"),
false, 1000, System.currentTimeMillis());
-        assertEquals(cfStore.metric.sstablesPerReadHistogram.max(), 5, 0.1);
+        assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.max(), 5, 0.1);
         int i = 0;
         for (Column c : cf.getSortedColumns())
         {
             assertEquals(ByteBufferUtil.string(c.name), "col" + (1000 + i++));
         }
         assertEquals(i, 500);
-        cfStore.metric.sstablesPerReadHistogram.clear();
+        cfStore.metric.sstablesPerReadHistogram.cf.clear();
         cf = cfStore.getColumnFamily(key, ByteBufferUtil.bytes("col1500"), ByteBufferUtil.bytes("col2000"),
false, 1000, System.currentTimeMillis());
-        assertEquals(cfStore.metric.sstablesPerReadHistogram.max(), 5, 0.1);
+        assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.max(), 5, 0.1);
 
         for (Column c : cf.getSortedColumns())
         {
@@ -485,9 +485,9 @@ public class KeyspaceTest extends SchemaLoader
         assertEquals(i, 1000);
 
         // reverse
-        cfStore.metric.sstablesPerReadHistogram.clear();
+        cfStore.metric.sstablesPerReadHistogram.cf.clear();
         cf = cfStore.getColumnFamily(key, ByteBufferUtil.bytes("col2000"), ByteBufferUtil.bytes("col1500"),
true, 1000, System.currentTimeMillis());
-        assertEquals(cfStore.metric.sstablesPerReadHistogram.max(), 5, 0.1);
+        assertEquals(cfStore.metric.sstablesPerReadHistogram.cf.max(), 5, 0.1);
         i = 500;
         for (Column c : cf.getSortedColumns())
         {
@@ -534,13 +534,13 @@ public class KeyspaceTest extends SchemaLoader
         }
         ByteBuffer start = ct.builder().add(ByteBufferUtil.bytes("a5")).add(ByteBufferUtil.bytes(85)).build();
         ByteBuffer finish = ct.builder().add(ByteBufferUtil.bytes("a5")).buildAsEndOfRange();
-        cfs.metric.sstablesPerReadHistogram.clear();
+        cfs.metric.sstablesPerReadHistogram.cf.clear();
         ColumnFamily cf = cfs.getColumnFamily(key, start, finish, false, 1000, System.currentTimeMillis());
         int colCount = 0;
         for (Column c : cf)
             colCount++;
         assertEquals(2, colCount);
-        assertEquals(2, cfs.metric.sstablesPerReadHistogram.max(), 0.1);
+        assertEquals(2, cfs.metric.sstablesPerReadHistogram.cf.max(), 0.1);
     }
 
     private void validateSliceLarge(ColumnFamilyStore cfStore) throws IOException


Mime
View raw message