cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [2/6] git commit: Add tombstone debug threshold and histogram patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057
Date Mon, 23 Sep 2013 18:33:53 GMT
Add tombstone debug threshold and histogram
patch by Russell Spitzer and Lyuben Todorov; reviewed by jbellis for CASSANDRA-6042 and CASSANDRA-6057


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2267c209
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2267c209
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2267c209

Branch: refs/heads/cassandra-2.0
Commit: 2267c20942feb486fbbf1bd54d7a0fbbe468ae10
Parents: 2980796
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Mon Sep 23 09:30:22 2013 -0500
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Mon Sep 23 12:39:46 2013 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 conf/cassandra.yaml                             |  5 ++++
 .../org/apache/cassandra/config/Config.java     |  2 ++
 .../cassandra/config/DatabaseDescriptor.java    | 14 ++++++++++
 .../apache/cassandra/db/ColumnFamilyStore.java  | 28 +++++++++++++++++---
 .../cassandra/db/ColumnFamilyStoreMBean.java    |  9 +++++++
 .../cassandra/db/filter/SliceQueryFilter.java   | 19 +++++++++++--
 .../cassandra/metrics/ColumnFamilyMetrics.java  |  8 ++++++
 .../cassandra/service/StorageService.java       | 10 +++++++
 .../cassandra/service/StorageServiceMBean.java  |  5 ++++
 10 files changed, 95 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 124b3e4..9ed00e2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,7 @@
  * Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
  * Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
  * Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
 
 
 1.2.10

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 27ac09b..a3cdec7 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -440,6 +440,11 @@ snapshot_before_compaction: false
 # lose data on truncation or drop.
 auto_snapshot: true
 
+# Log a debug message if more than this many tombstones are scanned
+# in a single-partition query.  Set the threshold on SliceQueryFilter
+# to debug to enable.
+tombstone_debug_threshold: 10000
+
 # Add column indexes to a row after its contents reach this size.
 # Increase if your column values are large, or if you have a very large
 # number of columns.  The competing causes are, Cassandra has to

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index a924a4c..292161b 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -176,6 +176,8 @@ public class Config
     private static boolean loadYaml = true;
     private static boolean outboundBindAny = false;
 
+    public volatile int tombstone_debug_threshold = 10000;
+
     public static boolean getOutboundBindAny()
     {
         return outboundBindAny;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 22de2d6..2c999ef 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -930,6 +930,20 @@ public class DatabaseDescriptor
     }
 
     /**
+     * How many tombstones need to be scanned before we log a
+     * debug message
+     */
+    public static int getTombstoneDebugThreshold()
+    {
+        return conf.tombstone_debug_threshold;
+    }
+
+    public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+    {
+        conf.tombstone_debug_threshold = tombstoneDebugThreshold;
+    }
+
+    /**
      * size of commitlog segments to allocate
      */
     public static int getCommitLogSegmentSize()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c646461..a41c157 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -49,10 +49,7 @@ import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.QueryPath;
+import org.apache.cassandra.db.filter.*;
 import org.apache.cassandra.db.index.SecondaryIndex;
 import org.apache.cassandra.db.index.SecondaryIndexManager;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -1221,6 +1218,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                 result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf,
gcBefore);
 
             }
+
+            if (filter.filter instanceof SliceQueryFilter)
+            {
+                // Log the number of tombstones scanned on single key queries
+                metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+                metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+            }
         }
         finally
         {
@@ -1907,6 +1911,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold()
<= 0;
     }
 
+    public long getTombstonesPerLastRead()
+    {
+        return metric.tombstoneScannedHistogram.count();
+    }
+
+    public float getPercentageTombstonesPerLastRead()
+    {
+        long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+        return (metric.tombstoneScannedHistogram.count() / total);
+    }
+
+    public long getLiveCellsPerLastRead()
+    {
+        return metric.liveScannedHistogram.count();
+    }
+
     // End JMX get/set.
 
     public long estimateKeys()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 554c204..f937032 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -286,6 +286,15 @@ public interface ColumnFamilyStoreMBean
      */
     public void disableAutoCompaction();
 
+    /** Number of tombstoned cells retreived during the last slicequery */
+    public long getTombstonesPerLastRead();
+
+    /** Percentage of tombstoned cells retreived during the last slicequery */
+    public float getPercentageTombstonesPerLastRead();
+
+    /** Number of live cells retreived during the last slicequery */
+    public long getLiveCellsPerLastRead();
+
     public long estimateKeys();
 
     /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
index 39d1ca1..7e1ec6b 100644
--- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
@@ -17,7 +17,9 @@
  */
 package org.apache.cassandra.db.filter;
 
-import java.io.*;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
 
@@ -26,9 +28,10 @@ import com.google.common.collect.Lists;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.*;
-import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.db.columniterator.ISSTableColumnIterator;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CompositeType;
@@ -172,6 +175,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
         }
 
         Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+        if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+            logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
     }
 
     public int getLiveCount(ColumnFamily cf)
@@ -249,6 +254,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
         return columnCounter == null ? 0 : columnCounter.live();
     }
 
+    public int lastIgnored()
+    {
+        return columnCounter == null ? 0 : columnCounter.ignored();
+    }
+
+    public int lastLive()
+    {
+        return columnCounter == null ? 0 : columnCounter.live();
+    }
+
     @Override
     public String toString()
     {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index d111667..9bd90a9 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -79,6 +79,10 @@ public class ColumnFamilyMetrics
     public final Gauge<Long> bloomFilterDiskSpaceUsed;
     /** Key cache hit rate  for this CF */
     public final Gauge<Double> keyCacheHitRate;
+    /** Tombstones scanned in queries on this CF */
+    public final Histogram tombstoneScannedHistogram;
+    /** Live cells scanned in queries on this CF */
+    public final Histogram liveScannedHistogram;
 
     private final MetricNameFactory factory;
 
@@ -295,6 +299,8 @@ public class ColumnFamilyMetrics
                 return Math.max(requests, 1); // to avoid NaN.
             }
         });
+        tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"));
+        liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"));
     }
 
     public void updateSSTableIterated(int count)
@@ -331,6 +337,8 @@ public class ColumnFamilyMetrics
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
     }
 
     class ColumnFamilyMetricNameFactory implements MetricNameFactory

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 7967dee..50719fd 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -3934,4 +3934,14 @@ public class StorageService extends NotificationBroadcasterSupport
implements IE
     {
         return DatabaseDescriptor.getPartitionerName();
     }
+
+    public int getTombstoneDebugThreshold()
+    {
+        return DatabaseDescriptor.getTombstoneDebugThreshold();
+    }
+
+    public void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+    {
+        DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold);
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/2267c209/src/java/org/apache/cassandra/service/StorageServiceMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
index 4fbed9c..63add12 100644
--- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java
+++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java
@@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter
     public String getClusterName();
     /** Returns the cluster partitioner */
     public String getPartitionerName();
+
+    /** Returns the threshold for returning debugging queries with many tombstones */
+    public int getTombstoneDebugThreshold();
+    /** Sets the threshold for returning debugging queries with many tombstones */
+    public void setTombstoneDebugThreshold(int tombstoneDebugThreshold);
 }


Mime
View raw message