cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmcken...@apache.org
Subject [18/22] cassandra git commit: 10585-3.0 patch
Date Fri, 04 Dec 2015 18:20:35 GMT
10585-3.0 patch


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5aa19cb6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5aa19cb6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5aa19cb6

Branch: refs/heads/cassandra-3.0
Commit: 5aa19cb68a7cbd95683db92a0c4bbcf47d4dcb23
Parents: b897e60
Author: Joshua McKenzie <jmckenzie@apache.org>
Authored: Fri Dec 4 13:19:07 2015 -0500
Committer: Joshua McKenzie <jmckenzie@apache.org>
Committed: Fri Dec 4 13:19:07 2015 -0500

----------------------------------------------------------------------
 .../db/SinglePartitionReadCommand.java          |  4 +-
 .../metrics/CASClientRequestMetrics.java        |  2 +-
 .../metrics/CassandraMetricsRegistry.java       | 10 ++--
 .../metrics/EstimatedHistogramReservoir.java    | 10 ++--
 .../cassandra/metrics/KeyspaceMetrics.java      |  8 +--
 .../apache/cassandra/metrics/TableMetrics.java  | 21 +++----
 .../cassandra/utils/EstimatedHistogram.java     | 31 +++++++---
 .../org/apache/cassandra/db/RowCacheTest.java   | 49 ++++++++++++++++
 .../cassandra/utils/EstimatedHistogramTest.java | 59 ++++++++++++++++++--
 9 files changed, 155 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index 4d410a0..de4c9c7 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -361,7 +361,9 @@ public class SinglePartitionReadCommand extends ReadCommand
             {
                 cfs.metric.rowCacheHit.inc();
                 Tracing.trace("Row cache hit");
-                return clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(), cachedPartition);
+                UnfilteredRowIterator unfilteredRowIterator = clusteringIndexFilter().getUnfilteredRowIterator(columnFilter(),
cachedPartition);
+                cfs.metric.updateSSTableIterated(0);
+                return unfilteredRowIterator;
             }
 
             cfs.metric.rowCacheHitOutOfRange.inc();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
index e6f2b81..4e64cff 100644
--- a/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/CASClientRequestMetrics.java
@@ -34,7 +34,7 @@ public class CASClientRequestMetrics extends ClientRequestMetrics
 
     public CASClientRequestMetrics(String scope) {
         super(scope);
-        contention = Metrics.histogram(factory.createMetricName("ContentionHistogram"));
+        contention = Metrics.histogram(factory.createMetricName("ContentionHistogram"), false);
         conditionNotMet =  Metrics.counter(factory.createMetricName("ConditionNotMet"));
         unfinishedCommit =  Metrics.counter(factory.createMetricName("UnfinishedCommit"));
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
index f7adb79..1cd3b6c 100644
--- a/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
+++ b/src/java/org/apache/cassandra/metrics/CassandraMetricsRegistry.java
@@ -72,24 +72,24 @@ public class CassandraMetricsRegistry extends MetricRegistry
         return meter;
     }
 
-    public Histogram histogram(MetricName name)
+    public Histogram histogram(MetricName name, boolean considerZeroes)
     {
-        Histogram histogram = register(name, new ClearableHistogram(new EstimatedHistogramReservoir()));
+        Histogram histogram = register(name, new ClearableHistogram(new EstimatedHistogramReservoir(considerZeroes)));
         registerMBean(histogram, name.getMBeanName());
 
         return histogram;
     }
 
-    public Histogram histogram(MetricName name, MetricName alias)
+    public Histogram histogram(MetricName name, MetricName alias, boolean considerZeroes)
     {
-        Histogram histogram = histogram(name);
+        Histogram histogram = histogram(name, considerZeroes);
         registerAlias(name, alias);
         return histogram;
     }
 
     public Timer timer(MetricName name)
     {
-        Timer timer = register(name, new Timer(new EstimatedHistogramReservoir()));
+        Timer timer = register(name, new Timer(new EstimatedHistogramReservoir(false)));
         registerMBean(timer, name.getMBeanName());
 
         return timer;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/metrics/EstimatedHistogramReservoir.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/EstimatedHistogramReservoir.java b/src/java/org/apache/cassandra/metrics/EstimatedHistogramReservoir.java
index 3051711..3658f3a 100644
--- a/src/java/org/apache/cassandra/metrics/EstimatedHistogramReservoir.java
+++ b/src/java/org/apache/cassandra/metrics/EstimatedHistogramReservoir.java
@@ -34,14 +34,14 @@ public class EstimatedHistogramReservoir implements Reservoir
     EstimatedHistogram histogram;
 
     // Default to >4 hours of in nanoseconds of buckets
-    public EstimatedHistogramReservoir()
+    public EstimatedHistogramReservoir(boolean considerZeroes)
     {
-        this(164);
+        this(164, considerZeroes);
     }
 
-    public EstimatedHistogramReservoir(int numBuckets)
+    public EstimatedHistogramReservoir(int numBuckets, boolean considerZeroes)
     {
-        histogram = new EstimatedHistogram(numBuckets);
+        histogram = new EstimatedHistogram(numBuckets, considerZeroes);
     }
 
     @Override
@@ -100,7 +100,7 @@ public class EstimatedHistogramReservoir implements Reservoir
         @Override
         public double getMean()
         {
-            return histogram.mean();
+            return histogram.rawMean();
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
index 62add07..ef62034 100644
--- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
@@ -224,10 +224,10 @@ public class KeyspaceMetrics
         writeLatency = new LatencyMetrics(factory, "Write");
         rangeLatency = new LatencyMetrics(factory, "Range");
         // create histograms for TableMetrics to replicate updates to
-        sstablesPerReadHistogram = Metrics.histogram(factory.createMetricName("SSTablesPerReadHistogram"));
-        tombstoneScannedHistogram = Metrics.histogram(factory.createMetricName("TombstoneScannedHistogram"));
-        liveScannedHistogram = Metrics.histogram(factory.createMetricName("LiveScannedHistogram"));
-        colUpdateTimeDeltaHistogram = Metrics.histogram(factory.createMetricName("ColUpdateTimeDeltaHistogram"));
+        sstablesPerReadHistogram = Metrics.histogram(factory.createMetricName("SSTablesPerReadHistogram"),
true);
+        tombstoneScannedHistogram = Metrics.histogram(factory.createMetricName("TombstoneScannedHistogram"),
false);
+        liveScannedHistogram = Metrics.histogram(factory.createMetricName("LiveScannedHistogram"),
false);
+        colUpdateTimeDeltaHistogram = Metrics.histogram(factory.createMetricName("ColUpdateTimeDeltaHistogram"),
false);
         viewLockAcquireTime =  Metrics.timer(factory.createMetricName("ViewLockAcquireTime"));
         viewReadTime = Metrics.timer(factory.createMetricName("ViewReadTime"));
         // add manually since histograms do not use createKeyspaceGauge method

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/metrics/TableMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index 3cd5b5b..85bf7f6 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -318,7 +318,7 @@ public class TableMetrics
                                                                  });
             }
         });
-        sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram);
+        sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram,
true);
         compressionRatio = createTableGauge("CompressionRatio", new Gauge<Double>()
         {
             public Double getValue()
@@ -610,12 +610,12 @@ public class TableMetrics
                 return Math.max(requests, 1); // to avoid NaN.
             }
         });
-        tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram);
-        liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram);
-        colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram",
cfs.keyspace.metric.colUpdateTimeDeltaHistogram);
+        tombstoneScannedHistogram = createTableHistogram("TombstoneScannedHistogram", cfs.keyspace.metric.tombstoneScannedHistogram,
false);
+        liveScannedHistogram = createTableHistogram("LiveScannedHistogram", cfs.keyspace.metric.liveScannedHistogram,
false);
+        colUpdateTimeDeltaHistogram = createTableHistogram("ColUpdateTimeDeltaHistogram",
cfs.keyspace.metric.colUpdateTimeDeltaHistogram, false);
         coordinatorReadLatency = Metrics.timer(factory.createMetricName("CoordinatorReadLatency"));
         coordinatorScanLatency = Metrics.timer(factory.createMetricName("CoordinatorScanLatency"));
-        waitingOnFreeMemtableSpace = Metrics.histogram(factory.createMetricName("WaitingOnFreeMemtableSpace"));
+        waitingOnFreeMemtableSpace = Metrics.histogram(factory.createMetricName("WaitingOnFreeMemtableSpace"),
false);
 
         // We do not want to capture view mutation specific metrics for a view
         // They only makes sense to capture on the base table
@@ -751,19 +751,20 @@ public class TableMetrics
      * Create a histogram-like interface that will register both a CF, keyspace and global
level
      * histogram and forward any updates to both
      */
-    protected TableHistogram createTableHistogram(String name, Histogram keyspaceHistogram)
+    protected TableHistogram createTableHistogram(String name, Histogram keyspaceHistogram,
boolean considerZeroes)
     {
-        return createTableHistogram(name, name, keyspaceHistogram);
+        return createTableHistogram(name, name, keyspaceHistogram, considerZeroes);
     }
 
-    protected TableHistogram createTableHistogram(String name, String alias, Histogram keyspaceHistogram)
+    protected TableHistogram createTableHistogram(String name, String alias, Histogram keyspaceHistogram,
boolean considerZeroes)
     {
-        Histogram cfHistogram = Metrics.histogram(factory.createMetricName(name), aliasFactory.createMetricName(alias));
+        Histogram cfHistogram = Metrics.histogram(factory.createMetricName(name), aliasFactory.createMetricName(alias),
considerZeroes);
         register(name, alias, cfHistogram);
         return new TableHistogram(cfHistogram,
                                   keyspaceHistogram,
                                   Metrics.histogram(globalFactory.createMetricName(name),
-                                                    globalAliasFactory.createMetricName(alias)));
+                                                    globalAliasFactory.createMetricName(alias),
+                                                    considerZeroes));
     }
 
     protected TableTimer createTableTimer(String name, Timer keyspaceTimer)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
index 89fbe4e..0249980 100644
--- a/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
+++ b/src/java/org/apache/cassandra/utils/EstimatedHistogram.java
@@ -56,7 +56,12 @@ public class EstimatedHistogram
 
     public EstimatedHistogram(int bucketCount)
     {
-        bucketOffsets = newOffsets(bucketCount);
+        this(bucketCount, false);
+    }
+
+    public EstimatedHistogram(int bucketCount, boolean considerZeroes)
+    {
+        bucketOffsets = newOffsets(bucketCount, considerZeroes);
         buckets = new AtomicLongArray(bucketOffsets.length + 1);
     }
 
@@ -67,12 +72,15 @@ public class EstimatedHistogram
         buckets = new AtomicLongArray(bucketData);
     }
 
-    private static long[] newOffsets(int size)
+    private static long[] newOffsets(int size, boolean considerZeroes)
     {
-        long[] result = new long[size];
+        long[] result = new long[size + (considerZeroes ? 1 : 0)];
+        int i = 0;
+        if (considerZeroes)
+            result[i++] = 0;
         long last = 1;
-        result[0] = last;
-        for (int i = 1; i < size; i++)
+        result[i++] = last;
+        for (; i < result.length; i++)
         {
             long next = Math.round(last * 1.2);
             if (next == last)
@@ -192,11 +200,20 @@ public class EstimatedHistogram
     }
 
     /**
-     * @return the mean histogram value (average of bucket offsets, weighted by count)
+     * @return the ceil of mean histogram value (average of bucket offsets, weighted by count)
      * @throws IllegalStateException if any values were greater than the largest bucket threshold
      */
     public long mean()
     {
+        return (long) Math.ceil(rawMean());
+    }
+
+    /**
+     * @return the mean histogram value (average of bucket offsets, weighted by count)
+     * @throws IllegalStateException if any values were greater than the largest bucket threshold
+     */
+    public double rawMean()
+    {
         int lastBucket = buckets.length() - 1;
         if (buckets.get(lastBucket) > 0)
             throw new IllegalStateException("Unable to compute ceiling for max when histogram
overflowed");
@@ -210,7 +227,7 @@ public class EstimatedHistogram
             sum += bCount * bucketOffsets[i];
         }
 
-        return (long) Math.ceil((double) sum / elements);
+        return (double) sum / elements;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/test/unit/org/apache/cassandra/db/RowCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/RowCacheTest.java b/test/unit/org/apache/cassandra/db/RowCacheTest.java
index b157adc..267e5e4 100644
--- a/test/unit/org/apache/cassandra/db/RowCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheTest.java
@@ -44,6 +44,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken;
 import org.apache.cassandra.locator.TokenMetadata;
+import org.apache.cassandra.metrics.ClearableHistogram;
 import org.apache.cassandra.schema.CachingParams;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.CacheService;
@@ -399,6 +400,54 @@ public class RowCacheTest
         cachedStore.truncateBlocking();
     }
 
+    @Test
+    public void testSSTablesPerReadHistogramWhenRowCache()
+    {
+        CompactionManager.instance.disableAutoCompaction();
+
+        Keyspace keyspace = Keyspace.open(KEYSPACE_CACHED);
+        ColumnFamilyStore cachedStore  = keyspace.getColumnFamilyStore(CF_CACHED);
+
+        // empty the row cache
+        CacheService.instance.invalidateRowCache();
+
+        // set global row cache size to 1 MB
+        CacheService.instance.setRowCacheCapacityInMB(1);
+
+        // inserting 100 rows into both column families
+        SchemaLoader.insertData(KEYSPACE_CACHED, CF_CACHED, 0, 100);
+
+        //force flush for confidence that SSTables exists
+        cachedStore.forceBlockingFlush();
+
+        ((ClearableHistogram)cachedStore.metric.sstablesPerReadHistogram.cf).clear();
+
+        for (int i = 0; i < 100; i++)
+        {
+            DecoratedKey key = Util.dk("key" + i);
+
+            Util.getAll(Util.cmd(cachedStore, key).build());
+
+            long count_before = cachedStore.metric.sstablesPerReadHistogram.cf.getCount();
+            Util.getAll(Util.cmd(cachedStore, key).build());
+
+            // check that SSTablePerReadHistogram has been updated by zero,
+            // so count has been increased and in a 1/2 of requests there were zero read
SSTables
+            long count_after = cachedStore.metric.sstablesPerReadHistogram.cf.getCount();
+            double belowMedian = cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getValue(0.49D);
+            double mean_after = cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMean();
+            assertEquals("SSTablePerReadHistogram should be updated even key found in row
cache", count_before + 1, count_after);
+            assertTrue("In half of requests we have not touched SSTables, " +
+                       "so 49 percentile (" + belowMedian + ") must be strongly less than
0.9", belowMedian < 0.9D);
+            assertTrue("In half of requests we have not touched SSTables, " +
+                       "so mean value (" + mean_after + ") must be strongly less than 1,
but greater than 0", mean_after < 0.999D && mean_after > 0.001D);
+        }
+
+        assertEquals("Min value of SSTablesPerRead should be zero", 0, cachedStore.metric.sstablesPerReadHistogram.cf.getSnapshot().getMin());
+
+        CacheService.instance.setRowCacheCapacityInMB(0);
+    }
+
     public void rowCacheLoad(int totalKeys, int keysToSave, int offset) throws Exception
     {
         CompactionManager.instance.disableAutoCompaction();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5aa19cb6/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java b/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java
index f813b9b..b3fbfb6 100644
--- a/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java
+++ b/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java
@@ -28,12 +28,23 @@ public class EstimatedHistogramTest
     @Test
     public void testSimple()
     {
-        // 0 and 1 map to the same, first bucket
-        EstimatedHistogram histogram = new EstimatedHistogram();
-        histogram.add(0);
-        assertEquals(1, histogram.get(0));
-        histogram.add(1);
-        assertEquals(2, histogram.get(0));
+        {
+            // 0 and 1 map to the same, first bucket
+            EstimatedHistogram histogram = new EstimatedHistogram();
+            histogram.add(0);
+            assertEquals(1, histogram.get(0));
+            histogram.add(1);
+            assertEquals(2, histogram.get(0));
+        }
+        {
+            // 0 and 1 map to different buckets
+            EstimatedHistogram histogram = new EstimatedHistogram(90, true);
+            histogram.add(0);
+            assertEquals(1, histogram.get(0));
+            histogram.add(1);
+            assertEquals(1, histogram.get(0));
+            assertEquals(1, histogram.get(1));
+        }
     }
 
     @Test
@@ -55,6 +66,33 @@ public class EstimatedHistogramTest
     }
 
     @Test
+    public void testMean()
+    {
+        {
+            EstimatedHistogram histogram = new EstimatedHistogram();
+            for (int i = 0; i < 40; i++)
+                histogram.add(0);
+            for (int i = 0; i < 20; i++)
+                histogram.add(1);
+            for (int i = 0; i < 10; i++)
+                histogram.add(2);
+            assertEquals(70, histogram.count());
+            assertEquals(2, histogram.mean());
+        }
+        {
+            EstimatedHistogram histogram = new EstimatedHistogram(90, true);
+            for (int i = 0; i < 40; i++)
+                histogram.add(0);
+            for (int i = 0; i < 20; i++)
+                histogram.add(1);
+            for (int i = 0; i < 10; i++)
+                histogram.add(2);
+            assertEquals(70, histogram.count());
+            assertEquals(1, histogram.mean());
+        }
+    }
+
+    @Test
     public void testFindingCorrectBuckets()
     {
         EstimatedHistogram histogram = new EstimatedHistogram();
@@ -119,5 +157,14 @@ public class EstimatedHistogramTest
             assertEquals(17, histogram.percentile(0.60));
             assertEquals(20, histogram.percentile(0.80));
         }
+        {
+            EstimatedHistogram histogram = new EstimatedHistogram(90, true);
+            histogram.add(0);
+            histogram.add(0);
+            histogram.add(1);
+
+            assertEquals(0, histogram.percentile(0.5));
+            assertEquals(1, histogram.percentile(0.99));
+        }
     }
 }


Mime
View raw message