Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D22C811D40 for ; Tue, 24 Jun 2014 18:08:59 +0000 (UTC) Received: (qmail 52019 invoked by uid 500); 24 Jun 2014 18:08:59 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 51894 invoked by uid 500); 24 Jun 2014 18:08:59 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 51488 invoked by uid 99); 24 Jun 2014 18:08:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Jun 2014 18:08:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id D963198C785; Tue, 24 Jun 2014 18:08:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: brandonwilliams@apache.org To: commits@cassandra.apache.org Date: Tue, 24 Jun 2014 18:09:01 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [04/16] git commit: Expose global CF metrics Expose global CF metrics Patch by Chris Lohfink, reviewed by brandonwilliams for CASSANDRA-7273 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e39e9e59 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e39e9e59 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e39e9e59 Branch: refs/heads/cassandra-2.0 Commit: e39e9e59309ee650caa210f8cae7e6b6d028a105 Parents: 93f54fc Author: Brandon Williams Authored: Tue Jun 24 13:04:17 2014 -0500 Committer: Brandon Williams Committed: Tue Jun 24 13:04:17 2014 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 4 +- src/java/org/apache/cassandra/db/Table.java | 2 +- .../cassandra/metrics/ColumnFamilyMetrics.java | 312 ++++++++++++++++--- .../cassandra/metrics/KeyspaceMetrics.java | 153 +++++---- .../cassandra/metrics/LatencyMetrics.java | 28 +- 6 files changed, 390 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d84f267..f837515 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 1.2.17 + * Expose global ColmunFamily metrics (CASSANDRA-7273) * Handle possible integer overflow in FastByteArrayOutputStream (CASSANDRA-7373) * cqlsh: 'ascii' values weren't formatted as text (CASSANDRA-7407) * cqlsh: ignore .cassandra permission errors (CASSANDRA-7266) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 1100fb9..c003f45 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1916,12 +1916,12 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean public double getTombstonesPerSlice() { - return metric.tombstoneScannedHistogram.getSnapshot().getMedian(); + return metric.tombstoneScannedHistogram.cf.getSnapshot().getMedian(); } public double getLiveCellsPerSlice() { - return metric.liveScannedHistogram.getSnapshot().getMedian(); + return metric.liveScannedHistogram.cf.getSnapshot().getMedian(); } // End JMX get/set. http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/src/java/org/apache/cassandra/db/Table.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Table.java b/src/java/org/apache/cassandra/db/Table.java index 611fd9a..939f800 100644 --- a/src/java/org/apache/cassandra/db/Table.java +++ b/src/java/org/apache/cassandra/db/Table.java @@ -269,12 +269,12 @@ public class Table for (int i = 0; i < indexLocks.length; i++) indexLocks[i] = new Object(); + this.metric = new KeyspaceMetrics(this); for (CFMetaData cfm : new ArrayList(Schema.instance.getTableDefinition(table).cfMetaData().values())) { logger.debug("Initializing {}.{}", name, cfm.cfName); initCf(cfm.cfId, cfm.cfName, loadSSTables); } - this.metric = new KeyspaceMetrics(this); } public void createReplicationStrategy(KSMetaData ksm) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java index a665365..ed3b6f9 100644 --- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java @@ -17,11 +17,14 @@ */ package org.apache.cassandra.metrics; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.Counter; -import com.yammer.metrics.core.Gauge; -import com.yammer.metrics.core.Histogram; -import com.yammer.metrics.core.MetricName; +import com.yammer.metrics.core.*; import com.yammer.metrics.util.RatioGauge; import org.apache.cassandra.db.ColumnFamilyStore; @@ -48,7 +51,7 @@ public class ColumnFamilyMetrics /** Histogram of estimated number of columns. */ public final Gauge estimatedColumnCountHistogram; /** Histogram of the number of sstable data files accessed per read */ - public final Histogram sstablesPerReadHistogram; + public final ColumnFamilyHistogram sstablesPerReadHistogram; /** Read metrics */ public final LatencyMetrics readLatency; /** Write metrics */ @@ -80,16 +83,31 @@ public class ColumnFamilyMetrics /** Key cache hit rate for this CF */ public final Gauge keyCacheHitRate; /** Tombstones scanned in queries on this CF */ - public final Histogram tombstoneScannedHistogram; + public final ColumnFamilyHistogram tombstoneScannedHistogram; /** Live cells scanned in queries on this CF */ - public final Histogram liveScannedHistogram; + public final ColumnFamilyHistogram liveScannedHistogram; private final MetricNameFactory factory; + private static final MetricNameFactory globalNameFactory = new AllColumnFamilyMetricNameFactory();; // for backward compatibility @Deprecated public final EstimatedHistogram sstablesPerRead = new EstimatedHistogram(35); @Deprecated public final EstimatedHistogram recentSSTablesPerRead = new EstimatedHistogram(35); - + + public final static LatencyMetrics globalReadLatency = new LatencyMetrics(globalNameFactory, "Read"); + public final static LatencyMetrics globalWriteLatency = new LatencyMetrics(globalNameFactory, "Write"); + public final static LatencyMetrics globalRangeLatency = new LatencyMetrics(globalNameFactory, "Range"); + + /** + * stores metrics that will be rolled into a single global metric + */ + public final static ConcurrentMap> allColumnFamilyMetrics = Maps.newConcurrentMap(); + + /** + * Stores all metric names created that can be used when unregistering + */ + public final static Set all = Sets.newHashSet(); + /** * Creates metrics for given {@link ColumnFamilyStore}. * @@ -99,21 +117,21 @@ public class ColumnFamilyMetrics { factory = new ColumnFamilyMetricNameFactory(cfs); - memtableColumnsCount = Metrics.newGauge(factory.createMetricName("MemtableColumnsCount"), new Gauge() + memtableColumnsCount = createColumnFamilyGauge("MemtableColumnsCount", new Gauge() { public Long value() { return cfs.getDataTracker().getMemtable().getOperations(); } }); - memtableDataSize = Metrics.newGauge(factory.createMetricName("MemtableDataSize"), new Gauge() + memtableDataSize = createColumnFamilyGauge("MemtableDataSize", new Gauge() { public Long value() { return cfs.getDataTracker().getMemtable().getLiveSize(); } }); - memtableSwitchCount = Metrics.newCounter(factory.createMetricName("MemtableSwitchCount")); + memtableSwitchCount = createColumnFamilyCounter("MemtableSwitchCount"); estimatedRowSizeHistogram = Metrics.newGauge(factory.createMetricName("EstimatedRowSizeHistogram"), new Gauge() { public long[] value() @@ -142,8 +160,8 @@ public class ColumnFamilyMetrics return histogram; } }); - sstablesPerReadHistogram = Metrics.newHistogram(factory.createMetricName("SSTablesPerReadHistogram"), true); - compressionRatio = Metrics.newGauge(factory.createMetricName("CompressionRatio"), new Gauge() + sstablesPerReadHistogram = createColumnFamilyHistogram("SSTablesPerReadHistogram", cfs.table.metric.sstablesPerReadHistogram); + compressionRatio = createColumnFamilyGauge("CompressionRatio", new Gauge() { public Double value() { @@ -157,11 +175,30 @@ public class ColumnFamilyMetrics total++; } } - return total != 0 ? (double)sum/total: 0; + return total != 0 ? sum / total : 0; + } + }, new Gauge() // global gauge + { + public Double value() + { + double sum = 0; + int total = 0; + for (Table keyspace : Table.all()) + { + for (SSTableReader sstable : keyspace.getAllSSTables()) + { + if (sstable.getCompressionRatio() != SSTableMetadata.NO_COMPRESSION_RATIO) + { + sum += sstable.getCompressionRatio(); + total++; + } + } + } + return total != 0 ? sum / total : 0; } }); - readLatency = new LatencyMetrics(factory, "Read"); - writeLatency = new LatencyMetrics(factory, "Write"); + readLatency = new LatencyMetrics(factory, "Read", cfs.table.metric.readLatency, globalReadLatency); + writeLatency = new LatencyMetrics(factory, "Write", cfs.table.metric.writeLatency, globalWriteLatency); pendingTasks = Metrics.newGauge(factory.createMetricName("PendingTasks"), new Gauge() { public Integer value() @@ -170,16 +207,16 @@ public class ColumnFamilyMetrics return Table.switchLock.getQueueLength(); } }); - liveSSTableCount = Metrics.newGauge(factory.createMetricName("LiveSSTableCount"), new Gauge() + liveSSTableCount = createColumnFamilyGauge("LiveSSTableCount", new Gauge() { public Integer value() { return cfs.getDataTracker().getSSTables().size(); } }); - liveDiskSpaceUsed = Metrics.newCounter(factory.createMetricName("LiveDiskSpaceUsed")); - totalDiskSpaceUsed = Metrics.newCounter(factory.createMetricName("TotalDiskSpaceUsed")); - minRowSize = Metrics.newGauge(factory.createMetricName("MinRowSize"), new Gauge() + liveDiskSpaceUsed = createColumnFamilyCounter("LiveDiskSpaceUsed"); + totalDiskSpaceUsed = createColumnFamilyCounter("TotalDiskSpaceUsed"); + minRowSize = createColumnFamilyGauge("MinRowSize", new Gauge() { public Long value() { @@ -191,8 +228,19 @@ public class ColumnFamilyMetrics } return min; } + }, new Gauge() // global gauge + { + public Long value() + { + long min = Long.MAX_VALUE; + for (Metric cfGauge : allColumnFamilyMetrics.get("MinRowSize")) + { + min = Math.min(min, ((Gauge) cfGauge).value().longValue()); + } + return min; + } }); - maxRowSize = Metrics.newGauge(factory.createMetricName("MaxRowSize"), new Gauge() + maxRowSize = createColumnFamilyGauge("MaxRowSize", new Gauge() { public Long value() { @@ -204,8 +252,19 @@ public class ColumnFamilyMetrics } return max; } + }, new Gauge() // global gauge + { + public Long value() + { + long max = 0; + for (Metric cfGauge : allColumnFamilyMetrics.get("MaxRowSize")) + { + max = Math.max(max, ((Gauge) cfGauge).value().longValue()); + } + return max; + } }); - meanRowSize = Metrics.newGauge(factory.createMetricName("MeanRowSize"), new Gauge() + meanRowSize = createColumnFamilyGauge("MeanRowSize", new Gauge() { public Long value() { @@ -219,8 +278,25 @@ public class ColumnFamilyMetrics } return count > 0 ? sum / count : 0; } + }, new Gauge() // global gauge + { + public Long value() + { + long sum = 0; + long count = 0; + for (Table keyspace : Table.all()) + { + for (SSTableReader sstable : keyspace.getAllSSTables()) + { + long n = sstable.getEstimatedRowSize().count(); + sum += sstable.getEstimatedRowSize().mean() * n; + count += n; + } + } + return count > 0 ? sum / count : 0; + } }); - bloomFilterFalsePositives = Metrics.newGauge(factory.createMetricName("BloomFilterFalsePositives"), new Gauge() + bloomFilterFalsePositives = createColumnFamilyGauge("BloomFilterFalsePositives", new Gauge() { public Long value() { @@ -230,23 +306,23 @@ public class ColumnFamilyMetrics return count; } }); - recentBloomFilterFalsePositives = Metrics.newGauge(factory.createMetricName("RecentBloomFilterFalsePositives"), new Gauge() + recentBloomFilterFalsePositives = createColumnFamilyGauge("RecentBloomFilterFalsePositives", new Gauge() { public Long value() { long count = 0L; - for (SSTableReader sstable: cfs.getSSTables()) + for (SSTableReader sstable : cfs.getSSTables()) count += sstable.getRecentBloomFilterFalsePositiveCount(); return count; } }); - bloomFilterFalseRatio = Metrics.newGauge(factory.createMetricName("BloomFilterFalseRatio"), new Gauge() + bloomFilterFalseRatio = createColumnFamilyGauge("BloomFilterFalseRatio", new Gauge() { public Double value() { long falseCount = 0L; long trueCount = 0L; - for (SSTableReader sstable: cfs.getSSTables()) + for (SSTableReader sstable : cfs.getSSTables()) { falseCount += sstable.getBloomFilterFalsePositiveCount(); trueCount += sstable.getBloomFilterTruePositiveCount(); @@ -255,8 +331,26 @@ public class ColumnFamilyMetrics return 0d; return (double) falseCount / (trueCount + falseCount); } + }, new Gauge() // global gauge + { + public Double value() + { + long falseCount = 0L; + long trueCount = 0L; + for (Table keyspace : Table.all()) + { + for (SSTableReader sstable : keyspace.getAllSSTables()) + { + falseCount += sstable.getBloomFilterFalsePositiveCount(); + trueCount += sstable.getBloomFilterTruePositiveCount(); + } + } + if (falseCount == 0L && trueCount == 0L) + return 0d; + return (double) falseCount / (trueCount + falseCount); + } }); - recentBloomFilterFalseRatio = Metrics.newGauge(factory.createMetricName("RecentBloomFilterFalseRatio"), new Gauge() + recentBloomFilterFalseRatio = createColumnFamilyGauge("RecentBloomFilterFalseRatio", new Gauge() { public Double value() { @@ -271,8 +365,26 @@ public class ColumnFamilyMetrics return 0d; return (double) falseCount / (trueCount + falseCount); } + }, new Gauge() // global gauge + { + public Double value() + { + long falseCount = 0L; + long trueCount = 0L; + for (Table keyspace : Table.all()) + { + for (SSTableReader sstable : keyspace.getAllSSTables()) + { + falseCount += sstable.getRecentBloomFilterFalsePositiveCount(); + trueCount += sstable.getRecentBloomFilterTruePositiveCount(); + } + } + if (falseCount == 0L && trueCount == 0L) + return 0d; + return (double) falseCount / (trueCount + falseCount); + } }); - bloomFilterDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("BloomFilterDiskSpaceUsed"), new Gauge() + bloomFilterDiskSpaceUsed = createColumnFamilyGauge("BloomFilterDiskSpaceUsed", new Gauge() { public Long value() { @@ -300,8 +412,8 @@ public class ColumnFamilyMetrics return Math.max(requests, 1); // to avoid NaN. } }); - tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"), true); - liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"), true); + tombstoneScannedHistogram = createColumnFamilyHistogram("TombstoneScannedHistogram", cfs.table.metric.tombstoneScannedHistogram); + liveScannedHistogram = createColumnFamilyHistogram("LiveScannedHistogram", cfs.table.metric.liveScannedHistogram); } public void updateSSTableIterated(int count) @@ -316,32 +428,121 @@ public class ColumnFamilyMetrics */ public void release() { + for(String name : all) + { + allColumnFamilyMetrics.get(name).remove(Metrics.defaultRegistry().allMetrics().get(factory.createMetricName(name))); + Metrics.defaultRegistry().removeMetric(factory.createMetricName(name)); + } readLatency.release(); writeLatency.release(); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableColumnsCount")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableDataSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableSwitchCount")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("CompressionRatio")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedRowSizeHistogram")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("EstimatedColumnCountHistogram")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("SSTablesPerReadHistogram")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("PendingTasks")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveSSTableCount")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveDiskSpaceUsed")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("TotalDiskSpaceUsed")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MinRowSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MaxRowSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MeanRowSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterFalsePositives")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalsePositives")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterFalseRatio")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram")); } + + /** + * Create a gauge that will be part of a merged version of all column families. The global gauge + * will merge each CF gauge by adding their values + */ + protected Gauge createColumnFamilyGauge(final String name, Gauge gauge) + { + return createColumnFamilyGauge(name, gauge, new Gauge() + { + public Long value() + { + long total = 0; + for (Metric cfGauge : allColumnFamilyMetrics.get(name)) + { + total = total + ((Gauge) cfGauge).value().longValue(); + } + return total; + } + }); + } + + /** + * Create a gauge that will be part of a merged version of all column families. The global gauge + * is defined as the globalGauge parameter + */ + protected Gauge createColumnFamilyGauge(String name, Gauge gauge, Gauge globalGauge) + { + Gauge cfGauge = Metrics.newGauge(factory.createMetricName(name), gauge); + if (register(name, cfGauge)) + { + Metrics.newGauge(globalNameFactory.createMetricName(name), globalGauge); + } + return cfGauge; + } + + /** + * Creates a counter that will also have a global counter thats the sum of all counters across + * different column families + */ + protected Counter createColumnFamilyCounter(final String name) + { + Counter cfCounter = Metrics.newCounter(factory.createMetricName(name)); + if (register(name, cfCounter)) + { + Metrics.newGauge(globalNameFactory.createMetricName(name), new Gauge() + { + public Long value() + { + long total = 0; + for (Metric cfGauge : allColumnFamilyMetrics.get(name)) + { + total += ((Counter) cfGauge).count(); + } + return total; + } + }); + } + return cfCounter; + } + + /** + * Create a histogram-like interface that will register both a CF, keyspace and global level + * histogram and forward any updates to both + */ + protected ColumnFamilyHistogram createColumnFamilyHistogram(String name, Histogram keyspaceHistogram) + { + Histogram cfHistogram = Metrics.newHistogram(factory.createMetricName(name), true); + register(name, cfHistogram); + return new ColumnFamilyHistogram(cfHistogram, keyspaceHistogram, Metrics.newHistogram(globalNameFactory.createMetricName(name), true)); + } + + /** + * Registers a metric to be removed when unloading CF. + * @return true if first time metric with that name has been registered + */ + private boolean register(String name, Metric metric) + { + boolean ret = allColumnFamilyMetrics.putIfAbsent(name, new HashSet()) == null; + allColumnFamilyMetrics.get(name).add(metric); + all.add(name); + return ret; + } + + public class ColumnFamilyHistogram + { + public final Histogram[] all; + public final Histogram cf; + private ColumnFamilyHistogram(Histogram cf, Histogram keyspace, Histogram global) + { + this.cf = cf; + this.all = new Histogram[]{cf, keyspace, global}; + } + + public void update(long i) + { + for(Histogram histo : all) + { + histo.update(i); + } + } + } + class ColumnFamilyMetricNameFactory implements MetricNameFactory { private final String keyspaceName; @@ -370,4 +571,17 @@ public class ColumnFamilyMetrics return new MetricName(groupName, type, metricName, keyspaceName + "." + columnFamilyName, mbeanName.toString()); } } + + static class AllColumnFamilyMetricNameFactory implements MetricNameFactory + { + public MetricName createMetricName(String metricName) + { + String groupName = ColumnFamilyMetrics.class.getPackage().getName(); + StringBuilder mbeanName = new StringBuilder(); + mbeanName.append(groupName).append(":"); + mbeanName.append("type=ColumnFamily"); + mbeanName.append(",name=").append(metricName); + return new MetricName(groupName, "ColumnFamily", metricName, "all", mbeanName.toString()); + } + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index 936187c..ccc9a49 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -18,12 +18,16 @@ package org.apache.cassandra.metrics; -import com.yammer.metrics.Metrics; -import com.yammer.metrics.core.*; +import java.util.Set; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Table; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.*; + /** * Metrics for {@link ColumnFamilyStore}. */ @@ -43,9 +47,25 @@ public class KeyspaceMetrics public final Gauge totalDiskSpaceUsed; /** Disk space used by bloom filter */ public final Gauge bloomFilterDiskSpaceUsed; - + /** (Local) read metrics */ + public final LatencyMetrics readLatency; + /** (Local) range slice metrics */ + public final LatencyMetrics rangeLatency; + /** (Local) write metrics */ + public final LatencyMetrics writeLatency; + /** Histogram of the number of sstable data files accessed per read */ + public final Histogram sstablesPerReadHistogram; + /** Tombstones scanned in queries on this Keyspace */ + public final Histogram tombstoneScannedHistogram; + /** Live cells scanned in queries on this Keyspace */ + public final Histogram liveScannedHistogram; + private final MetricNameFactory factory; - + private Table keyspace; + + /** set containing names of all the metrics stored here, for releasing later */ + private Set allMetrics = Sets.newHashSet(); + /** * Creates metrics for given {@link ColumnFamilyStore}. * @@ -54,39 +74,26 @@ public class KeyspaceMetrics public KeyspaceMetrics(final Table ks) { factory = new KeyspaceMetricNameFactory(ks); - - memtableColumnsCount = Metrics.newGauge(factory.createMetricName("MemtableColumnsCount"), new Gauge() + keyspace = ks; + memtableColumnsCount = createKeyspaceGauge("MemtableColumnsCount", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long total = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - { - total += cf.metric.memtableColumnsCount.value(); - } - return total; + return metric.memtableColumnsCount.value(); } }); - memtableDataSize = Metrics.newGauge(factory.createMetricName("MemtableDataSize"), new Gauge() + memtableDataSize = createKeyspaceGauge("MemtableDataSize", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long total = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - { - total += cf.metric.memtableDataSize.value(); - } - return total; + return metric.memtableDataSize.value(); } }); - memtableSwitchCount = Metrics.newGauge(factory.createMetricName("MemtableSwitchCount"), new Gauge() + memtableSwitchCount = createKeyspaceGauge("MemtableSwitchCount", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long sum = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - sum += cf.metric.memtableSwitchCount.count(); - return sum; + return metric.memtableSwitchCount.count(); } }); pendingTasks = Metrics.newGauge(factory.createMetricName("PendingTasks"), new Gauge() @@ -96,40 +103,37 @@ public class KeyspaceMetrics return Table.switchLock.getQueueLength(); } }); - liveDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("LiveDiskSpaceUsed"), new Gauge() + liveDiskSpaceUsed = createKeyspaceGauge("LiveDiskSpaceUsed", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long sum = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - { - sum += cf.metric.liveDiskSpaceUsed.count(); - } - return sum; + return metric.liveDiskSpaceUsed.count(); } }); - totalDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("TotalDiskSpaceUsed"), new Gauge() + totalDiskSpaceUsed = createKeyspaceGauge("TotalDiskSpaceUsed", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long sum = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - { - sum += cf.metric.totalDiskSpaceUsed.count(); - } - return sum; + return metric.totalDiskSpaceUsed.count(); } }); - bloomFilterDiskSpaceUsed = Metrics.newGauge(factory.createMetricName("BloomFilterDiskSpaceUsed"), new Gauge() + bloomFilterDiskSpaceUsed = createKeyspaceGauge("BloomFilterDiskSpaceUsed", new MetricValue() { - public Long value() + public Long getValue(ColumnFamilyMetrics metric) { - long total = 0; - for (ColumnFamilyStore cf : ks.getColumnFamilyStores()) - total += cf.metric.bloomFilterDiskSpaceUsed.value(); - return total; + return metric.bloomFilterDiskSpaceUsed.value(); } }); + // latency metrics for ColumnFamilyMetrics to update + readLatency = new LatencyMetrics(factory, "Read"); + writeLatency = new LatencyMetrics(factory, "Write"); + rangeLatency = new LatencyMetrics(factory, "Range"); + // create histograms for ColumnFamilyMetrics to replicate updates to + sstablesPerReadHistogram = Metrics.newHistogram(factory.createMetricName("SSTablesPerReadHistogram"), true); + tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"), true); + liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"), true); + // add manually since histograms do not use createKeyspaceGauge method + allMetrics.addAll(Lists.newArrayList("SSTablesPerReadHistogram", "TombstoneScannedHistogram", "LiveScannedHistogram")); } /** @@ -137,14 +141,51 @@ public class KeyspaceMetrics */ public void release() { - Metrics.defaultRegistry().removeMetric(factory.createMetricName("AllMemtablesDataSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableColumnsCount")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableDataSize")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("MemtableSwitchCount")); + for(String name : allMetrics) + { + Metrics.defaultRegistry().removeMetric(factory.createMetricName(name)); + } + // latency metrics contain multiple metrics internally and need to be released manually + readLatency.release(); + writeLatency.release(); + rangeLatency.release(); Metrics.defaultRegistry().removeMetric(factory.createMetricName("PendingTasks")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveDiskSpaceUsed")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("TotalDiskSpaceUsed")); - Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed")); + } + + /** + * Represents a column family metric value. + */ + private interface MetricValue + { + /** + * get value of a metric + * @param columnfamilymetrics of a column family in this keyspace + * @return current value of a metric + */ + public Long getValue(ColumnFamilyMetrics metric); + } + + /** + * Creates a gauge that will sum the current value of a metric for all column families in this keyspace + * @param name + * @param MetricValue + * @return Gauge>Long> that computes sum of MetricValue.getValue() + */ + private Gauge createKeyspaceGauge(String name, final MetricValue extractor) + { + allMetrics.add(name); + return Metrics.newGauge(factory.createMetricName(name), new Gauge() + { + public Long value() + { + long sum = 0; + for (ColumnFamilyStore cf : keyspace.getColumnFamilyStores()) + { + sum += extractor.getValue(cf.metric); + } + return sum; + } + }); } class KeyspaceMetricNameFactory implements MetricNameFactory http://git-wip-us.apache.org/repos/asf/cassandra/blob/e39e9e59/src/java/org/apache/cassandra/metrics/LatencyMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java index c2c48f2..5928cc9 100644 --- a/src/java/org/apache/cassandra/metrics/LatencyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/LatencyMetrics.java @@ -17,14 +17,17 @@ */ package org.apache.cassandra.metrics; +import java.util.List; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.utils.EstimatedHistogram; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Counter; import com.yammer.metrics.core.Timer; -import org.apache.cassandra.utils.EstimatedHistogram; - /** * Metrics about latencies */ @@ -35,6 +38,9 @@ public class LatencyMetrics /** Total latency in micro sec */ public final Counter totalLatency; + /** parent metrics to replicate any updates to **/ + private List parents = Lists.newArrayList(); + protected final MetricNameFactory factory; protected final String namePrefix; @@ -80,6 +86,20 @@ public class LatencyMetrics latency = Metrics.newTimer(factory.createMetricName(namePrefix + "Latency"), TimeUnit.MICROSECONDS, TimeUnit.SECONDS); totalLatency = Metrics.newCounter(factory.createMetricName(namePrefix + "TotalLatency")); } + + /** + * Create LatencyMetrics with given group, type, prefix to append to each metric name, and scope. Any updates + * to this will also run on parent + * + * @param factory MetricName factory to use + * @param namePrefix Prefix to append to each metric name + * @param parents... any amount of parents to replicate updates to + */ + public LatencyMetrics(MetricNameFactory factory, String namePrefix, LatencyMetrics ... parents) + { + this(factory, namePrefix); + this.parents.addAll(ImmutableList.copyOf(parents)); + } /** takes nanoseconds **/ public void addNano(long nanos) @@ -97,6 +117,10 @@ public class LatencyMetrics } totalLatencyHistogram.add(micros); recentLatencyHistogram.add(micros); + for(LatencyMetrics parent : parents) + { + parent.addMicro(micros); + } } public void release()