cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject cassandra git commit: Emit metrics whenever we hit tombstone failures and warn thresholds
Date Fri, 08 Sep 2017 10:40:33 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 6c0e6693f -> 34dec4c79


Emit metrics whenever we hit tombstone failures and warn thresholds

Patch by Sarath P S; reviewed by marcuse for CASSANDRA-13771


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/34dec4c7
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/34dec4c7
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/34dec4c7

Branch: refs/heads/trunk
Commit: 34dec4c791dc86105af906fd649897484d4735d8
Parents: 6c0e669
Author: Sarath P S <sps22@apple.com>
Authored: Tue Sep 5 14:31:29 2017 +0530
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Fri Sep 8 12:37:12 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../org/apache/cassandra/db/ReadCommand.java    |  6 ++
 .../apache/cassandra/metrics/TableMetrics.java  | 13 +++
 .../miscellaneous/TombstonesTest.java           | 95 ++++++++++++++++----
 4 files changed, 99 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index da5d153..8311162 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 4.0
+ * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771)
  * Make netty EventLoopGroups daemon threads (CASSANDRA-13837)
  * Race condition when closing stream sessions (CASSANDRA-13852)
  * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/src/java/org/apache/cassandra/db/ReadCommand.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index 08ebc86..d094986 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -454,6 +454,7 @@ public abstract class ReadCommand extends MonitorableImpl implements ReadQuery
                 {
                     String query = ReadCommand.this.toCQLString();
                     Tracing.trace("Scanned over {} tombstones for query {}; query aborted
(see tombstone_failure_threshold)", failureThreshold, query);
+                    metric.tombstoneFailures.inc();
                     throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(),
currentKey, clustering);
                 }
             }
@@ -471,6 +472,11 @@ public abstract class ReadCommand extends MonitorableImpl implements
ReadQuery
                 {
                     String msg = String.format("Read %d live rows and %d tombstone cells
for query %1.512s (see tombstone_warn_threshold)", liveRows, tombstones, ReadCommand.this.toCQLString());
                     ClientWarn.instance.warn(msg);
+                    if (tombstones < failureThreshold)
+                    {
+                        metric.tombstoneWarnings.inc();
+                    }
+
                     logger.warn(msg);
                 }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/src/java/org/apache/cassandra/metrics/TableMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index 7e6ca25..04fbf46 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -141,6 +141,14 @@ public class TableMetrics
     public final Counter rowCacheHit;
     /** Number of row cache misses */
     public final Counter rowCacheMiss;
+    /**
+     * Number of tombstone read failures
+     */
+    public final Counter tombstoneFailures;
+    /**
+     * Number of tombstone read warnings
+     */
+    public final Counter tombstoneWarnings;
     /** CAS Prepare metrics */
     public final LatencyMetrics casPrepare;
     /** CAS Propose metrics */
@@ -515,6 +523,7 @@ public class TableMetrics
         rangeLatency = new LatencyMetrics(factory, "Range", cfs.keyspace.metric.rangeLatency,
globalRangeLatency);
         pendingFlushes = createTableCounter("PendingFlushes");
         bytesFlushed = createTableCounter("BytesFlushed");
+
         compactionBytesWritten = createTableCounter("CompactionBytesWritten");
         pendingCompactions = createTableGauge("PendingCompactions", new Gauge<Integer>()
         {
@@ -806,6 +815,10 @@ public class TableMetrics
         rowCacheHitOutOfRange = createTableCounter("RowCacheHitOutOfRange");
         rowCacheHit = createTableCounter("RowCacheHit");
         rowCacheMiss = createTableCounter("RowCacheMiss");
+
+        tombstoneFailures = createTableCounter("TombstoneFailures");
+        tombstoneWarnings = createTableCounter("TombstoneWarnings");
+
         droppedMutations = createTableCounter("DroppedMutations");
 
         casPrepare = new LatencyMetrics(factory, "CasPrepare", cfs.keyspace.metric.casPrepare);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
index 72ed887..85048ae 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java
@@ -24,10 +24,14 @@ import com.google.common.base.Throwables;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.filter.TombstoneOverwhelmingException;
 
+import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
 import static junit.framework.Assert.fail;
 
@@ -36,34 +40,45 @@ import static junit.framework.Assert.fail;
  */
 public class TombstonesTest extends CQLTester
 {
-    static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
-    static final int THRESHOLD = 100;
+    static final int ORIGINAL_FAILURE_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+    static final int FAILURE_THRESHOLD = 100;
+
+    static final int ORIGINAL_WARN_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold();
+    static final int WARN_THRESHOLD = 50;
 
     @BeforeClass
     public static void setUp() throws Throwable
     {
         DatabaseDescriptor.daemonInitialization();
-        DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD);
+        DatabaseDescriptor.setTombstoneFailureThreshold(FAILURE_THRESHOLD);
+        DatabaseDescriptor.setTombstoneWarnThreshold(WARN_THRESHOLD);
     }
 
     @AfterClass
     public static void tearDown()
     {
-        DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD);
+        DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_FAILURE_THRESHOLD);
+        DatabaseDescriptor.setTombstoneWarnThreshold(ORIGINAL_WARN_THRESHOLD);
     }
 
     @Test
     public void testBelowThresholdSelect() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a, b));");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
 
         // insert exactly the amount of tombstones that shouldn't trigger an exception
-        for (int i = 0; i < THRESHOLD; i++)
+        for (int i = 0; i < FAILURE_THRESHOLD; i++)
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
 
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
+            assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
         }
         catch (Throwable e)
         {
@@ -74,10 +89,13 @@ public class TombstonesTest extends CQLTester
     @Test
     public void testBeyondThresholdSelect() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a, b));");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
 
         // insert exactly the amount of tombstones that *SHOULD* trigger an exception
-        for (int i = 0; i < THRESHOLD + 1; i++)
+        for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
 
         try
@@ -88,19 +106,24 @@ public class TombstonesTest extends CQLTester
         catch (Throwable e)
         {
             String error = "Expected exception instanceof TombstoneOverwhelmingException
instead got "
-                          + System.lineSeparator()
-                          + Throwables.getStackTraceAsString(e);
+                           + System.lineSeparator()
+                           + Throwables.getStackTraceAsString(e);
             assertTrue(error, e instanceof TombstoneOverwhelmingException);
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
+            assertEquals(oldFailures + 1, cfs.metric.tombstoneFailures.getCount());
         }
     }
 
     @Test
     public void testAllShadowedSelect() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a, b));");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
 
         // insert exactly the amount of tombstones that *SHOULD* normally trigger an exception
-        for (int i = 0; i < THRESHOLD + 1; i++)
+        for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
 
         // delete all with a partition level tombstone
@@ -109,6 +132,8 @@ public class TombstonesTest extends CQLTester
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
+            assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
         }
         catch (Throwable e)
         {
@@ -119,9 +144,12 @@ public class TombstonesTest extends CQLTester
     @Test
     public void testLiveShadowedCellsSelect() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));");
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a, b));");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
 
-        for (int i = 0; i < THRESHOLD + 1; i++)
+        for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', 'column');");
 
         // delete all with a partition level tombstone
@@ -130,6 +158,8 @@ public class TombstonesTest extends CQLTester
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
+            assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
         }
         catch (Throwable e)
         {
@@ -140,9 +170,12 @@ public class TombstonesTest extends CQLTester
     @Test
     public void testExpiredTombstones() throws Throwable
     {
-        createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds
= 1;");
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a, b)) WITH gc_grace_seconds = 1;");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
 
-        for (int i = 0; i < THRESHOLD + 1; i++)
+        for (int i = 0; i < FAILURE_THRESHOLD + 1; i++)
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
 
         // not yet past gc grace - must throw a TOE
@@ -154,6 +187,9 @@ public class TombstonesTest extends CQLTester
         catch (Throwable e)
         {
             assertTrue(e instanceof TombstoneOverwhelmingException);
+
+            assertEquals(++oldFailures, cfs.metric.tombstoneFailures.getCount());
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
         }
 
         // sleep past gc grace
@@ -163,10 +199,37 @@ public class TombstonesTest extends CQLTester
         try
         {
             execute("SELECT * FROM %s WHERE a = 'key';");
+            assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+            assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount());
         }
         catch (Throwable e)
         {
             fail("SELECT with expired tombstones beyond the threshold should not have failed,
but has: " + e);
         }
     }
+
+    @Test
+    public void testBeyondWarnThresholdSelect() throws Throwable
+    {
+        String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY
KEY (a,b));");
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName);
+        long oldFailures = cfs.metric.tombstoneFailures.getCount();
+        long oldWarnings = cfs.metric.tombstoneWarnings.getCount();
+
+        // insert the number of tombstones that *SHOULD* trigger an Warning
+        for (int i = 0; i < WARN_THRESHOLD + 1; i++)
+            execute("INSERT INTO %s (a, b, c ) VALUES ('key', 'cc" + i + "',  null);");
+        try
+        {
+            execute("SELECT * FROM %s WHERE a = 'key';");
+            assertEquals(oldWarnings + 1, cfs.metric.tombstoneWarnings.getCount());
+            assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount());
+        }
+        catch (Throwable e)
+        {
+            fail("SELECT with tombstones below the failure threshold and above warning threashhold
should not have failed, but has: " + e);
+        }
+    }
+
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message