Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 23AB0200CFE for ; Fri, 8 Sep 2017 12:40:35 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 21EBB160C54; Fri, 8 Sep 2017 10:40:35 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1885E1609DA for ; Fri, 8 Sep 2017 12:40:33 +0200 (CEST) Received: (qmail 94429 invoked by uid 500); 8 Sep 2017 10:40:33 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 94417 invoked by uid 99); 8 Sep 2017 10:40:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 08 Sep 2017 10:40:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 090A8F5533; Fri, 8 Sep 2017 10:40:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: marcuse@apache.org To: commits@cassandra.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: cassandra git commit: Emit metrics whenever we hit tombstone failures and warn thresholds Date: Fri, 8 Sep 2017 10:40:33 +0000 (UTC) archived-at: Fri, 08 Sep 2017 10:40:35 -0000 Repository: cassandra Updated Branches: refs/heads/trunk 6c0e6693f -> 34dec4c79 Emit metrics whenever we hit tombstone failures and warn thresholds Patch by Sarath P S; reviewed by marcuse for CASSANDRA-13771 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/34dec4c7 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/34dec4c7 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/34dec4c7 Branch: refs/heads/trunk Commit: 34dec4c791dc86105af906fd649897484d4735d8 Parents: 6c0e669 Author: Sarath P S Authored: Tue Sep 5 14:31:29 2017 +0530 Committer: Marcus Eriksson Committed: Fri Sep 8 12:37:12 2017 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../org/apache/cassandra/db/ReadCommand.java | 6 ++ .../apache/cassandra/metrics/TableMetrics.java | 13 +++ .../miscellaneous/TombstonesTest.java | 95 ++++++++++++++++---- 4 files changed, 99 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index da5d153..8311162 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771) * Make netty EventLoopGroups daemon threads (CASSANDRA-13837) * Race condition when closing stream sessions (CASSANDRA-13852) * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831) http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/src/java/org/apache/cassandra/db/ReadCommand.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index 08ebc86..d094986 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -454,6 +454,7 @@ public abstract class ReadCommand extends MonitorableImpl implements ReadQuery { String query = ReadCommand.this.toCQLString(); Tracing.trace("Scanned over {} tombstones for query {}; query aborted (see tombstone_failure_threshold)", failureThreshold, query); + metric.tombstoneFailures.inc(); throw new TombstoneOverwhelmingException(tombstones, query, ReadCommand.this.metadata(), currentKey, clustering); } } @@ -471,6 +472,11 @@ public abstract class ReadCommand extends MonitorableImpl implements ReadQuery { String msg = String.format("Read %d live rows and %d tombstone cells for query %1.512s (see tombstone_warn_threshold)", liveRows, tombstones, ReadCommand.this.toCQLString()); ClientWarn.instance.warn(msg); + if (tombstones < failureThreshold) + { + metric.tombstoneWarnings.inc(); + } + logger.warn(msg); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/src/java/org/apache/cassandra/metrics/TableMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 7e6ca25..04fbf46 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -141,6 +141,14 @@ public class TableMetrics public final Counter rowCacheHit; /** Number of row cache misses */ public final Counter rowCacheMiss; + /** + * Number of tombstone read failures + */ + public final Counter tombstoneFailures; + /** + * Number of tombstone read warnings + */ + public final Counter tombstoneWarnings; /** CAS Prepare metrics */ public final LatencyMetrics casPrepare; /** CAS Propose metrics */ @@ -515,6 +523,7 @@ public class TableMetrics rangeLatency = new LatencyMetrics(factory, "Range", cfs.keyspace.metric.rangeLatency, globalRangeLatency); pendingFlushes = createTableCounter("PendingFlushes"); bytesFlushed = createTableCounter("BytesFlushed"); + compactionBytesWritten = createTableCounter("CompactionBytesWritten"); pendingCompactions = createTableGauge("PendingCompactions", new Gauge() { @@ -806,6 +815,10 @@ public class TableMetrics rowCacheHitOutOfRange = createTableCounter("RowCacheHitOutOfRange"); rowCacheHit = createTableCounter("RowCacheHit"); rowCacheMiss = createTableCounter("RowCacheMiss"); + + tombstoneFailures = createTableCounter("TombstoneFailures"); + tombstoneWarnings = createTableCounter("TombstoneWarnings"); + droppedMutations = createTableCounter("DroppedMutations"); casPrepare = new LatencyMetrics(factory, "CasPrepare", cfs.keyspace.metric.casPrepare); http://git-wip-us.apache.org/repos/asf/cassandra/blob/34dec4c7/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java index 72ed887..85048ae 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/TombstonesTest.java @@ -24,10 +24,14 @@ import com.google.common.base.Throwables; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; +import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; import static junit.framework.Assert.fail; @@ -36,34 +40,45 @@ import static junit.framework.Assert.fail; */ public class TombstonesTest extends CQLTester { - static final int ORIGINAL_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold(); - static final int THRESHOLD = 100; + static final int ORIGINAL_FAILURE_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold(); + static final int FAILURE_THRESHOLD = 100; + + static final int ORIGINAL_WARN_THRESHOLD = DatabaseDescriptor.getTombstoneFailureThreshold(); + static final int WARN_THRESHOLD = 50; @BeforeClass public static void setUp() throws Throwable { DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setTombstoneFailureThreshold(THRESHOLD); + DatabaseDescriptor.setTombstoneFailureThreshold(FAILURE_THRESHOLD); + DatabaseDescriptor.setTombstoneWarnThreshold(WARN_THRESHOLD); } @AfterClass public static void tearDown() { - DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_THRESHOLD); + DatabaseDescriptor.setTombstoneFailureThreshold(ORIGINAL_FAILURE_THRESHOLD); + DatabaseDescriptor.setTombstoneWarnThreshold(ORIGINAL_WARN_THRESHOLD); } @Test public void testBelowThresholdSelect() throws Throwable { - createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); // insert exactly the amount of tombstones that shouldn't trigger an exception - for (int i = 0; i < THRESHOLD; i++) + for (int i = 0; i < FAILURE_THRESHOLD; i++) execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); try { execute("SELECT * FROM %s WHERE a = 'key';"); + assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount()); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); } catch (Throwable e) { @@ -74,10 +89,13 @@ public class TombstonesTest extends CQLTester @Test public void testBeyondThresholdSelect() throws Throwable { - createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); // insert exactly the amount of tombstones that *SHOULD* trigger an exception - for (int i = 0; i < THRESHOLD + 1; i++) + for (int i = 0; i < FAILURE_THRESHOLD + 1; i++) execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); try @@ -88,19 +106,24 @@ public class TombstonesTest extends CQLTester catch (Throwable e) { String error = "Expected exception instanceof TombstoneOverwhelmingException instead got " - + System.lineSeparator() - + Throwables.getStackTraceAsString(e); + + System.lineSeparator() + + Throwables.getStackTraceAsString(e); assertTrue(error, e instanceof TombstoneOverwhelmingException); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); + assertEquals(oldFailures + 1, cfs.metric.tombstoneFailures.getCount()); } } @Test public void testAllShadowedSelect() throws Throwable { - createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); // insert exactly the amount of tombstones that *SHOULD* normally trigger an exception - for (int i = 0; i < THRESHOLD + 1; i++) + for (int i = 0; i < FAILURE_THRESHOLD + 1; i++) execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); // delete all with a partition level tombstone @@ -109,6 +132,8 @@ public class TombstonesTest extends CQLTester try { execute("SELECT * FROM %s WHERE a = 'key';"); + assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount()); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); } catch (Throwable e) { @@ -119,9 +144,12 @@ public class TombstonesTest extends CQLTester @Test public void testLiveShadowedCellsSelect() throws Throwable { - createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b));"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); - for (int i = 0; i < THRESHOLD + 1; i++) + for (int i = 0; i < FAILURE_THRESHOLD + 1; i++) execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', 'column');"); // delete all with a partition level tombstone @@ -130,6 +158,8 @@ public class TombstonesTest extends CQLTester try { execute("SELECT * FROM %s WHERE a = 'key';"); + assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount()); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); } catch (Throwable e) { @@ -140,9 +170,12 @@ public class TombstonesTest extends CQLTester @Test public void testExpiredTombstones() throws Throwable { - createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;"); + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds = 1;"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); - for (int i = 0; i < THRESHOLD + 1; i++) + for (int i = 0; i < FAILURE_THRESHOLD + 1; i++) execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);"); // not yet past gc grace - must throw a TOE @@ -154,6 +187,9 @@ public class TombstonesTest extends CQLTester catch (Throwable e) { assertTrue(e instanceof TombstoneOverwhelmingException); + + assertEquals(++oldFailures, cfs.metric.tombstoneFailures.getCount()); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); } // sleep past gc grace @@ -163,10 +199,37 @@ public class TombstonesTest extends CQLTester try { execute("SELECT * FROM %s WHERE a = 'key';"); + assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount()); + assertEquals(oldWarnings, cfs.metric.tombstoneWarnings.getCount()); } catch (Throwable e) { fail("SELECT with expired tombstones beyond the threshold should not have failed, but has: " + e); } } + + @Test + public void testBeyondWarnThresholdSelect() throws Throwable + { + String tableName = createTable("CREATE TABLE %s (a text, b text, c text, PRIMARY KEY (a,b));"); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(tableName); + long oldFailures = cfs.metric.tombstoneFailures.getCount(); + long oldWarnings = cfs.metric.tombstoneWarnings.getCount(); + + // insert the number of tombstones that *SHOULD* trigger an Warning + for (int i = 0; i < WARN_THRESHOLD + 1; i++) + execute("INSERT INTO %s (a, b, c ) VALUES ('key', 'cc" + i + "', null);"); + try + { + execute("SELECT * FROM %s WHERE a = 'key';"); + assertEquals(oldWarnings + 1, cfs.metric.tombstoneWarnings.getCount()); + assertEquals(oldFailures, cfs.metric.tombstoneFailures.getCount()); + } + catch (Throwable e) + { + fail("SELECT with tombstones below the failure threshold and above warning threashhold should not have failed, but has: " + e); + } + } + + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org For additional commands, e-mail: commits-help@cassandra.apache.org