cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [5/8] cassandra git commit: Warn when an extra-large partition is compacted, CASSANDRA-9643
Date Thu, 02 Jul 2015 18:58:33 GMT
Warn when an extra-large partition is compacted, CASSANDRA-9643


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/492c65a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/492c65a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/492c65a6

Branch: refs/heads/trunk
Commit: 492c65a677c02727c6186be801eb81763b7a312d
Parents: b19dd38
Author: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Authored: Tue Jun 30 17:12:56 2015 +0800
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Thu Jul 2 11:57:30 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                          |  1 +
 conf/cassandra.yaml                                  |  3 +++
 src/java/org/apache/cassandra/config/Config.java     |  1 +
 .../apache/cassandra/config/DatabaseDescriptor.java  |  2 ++
 .../io/sstable/format/big/BigTableWriter.java        | 15 ++++++++++++++-
 5 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0b38ff0..4042722 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.2.0-rc2
+ * Warn when an extra-large partition is compacted (CASSANDRA-9643)
  * (cqlsh) Allow setting the initial connection timeout (CASSANDRA-9601)
  * BulkLoader has --transport-factory option but does not use it (CASSANDRA-9675)
  * Allow JMX over SSL directly from nodetool (CASSANDRA-9090)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index dba8e1f..640af8f 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -622,6 +622,9 @@ batch_size_fail_threshold_in_kb: 50
 # of compaction, including validation compaction.
 compaction_throughput_mb_per_sec: 16
 
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold_mb: 100
+
 # When compacting, the replacement sstable(s) can be opened before they
 # are completely written, and used in place of the prior sstables for
 # any range that has been written. This helps to smoothly transfer reads 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 5beeef2..9f37d9a 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -152,6 +152,7 @@ public class Config
     public volatile int batch_size_fail_threshold_in_kb = 50;
     public Integer concurrent_compactors;
     public volatile Integer compaction_throughput_mb_per_sec = 16;
+    public volatile Integer compaction_large_partition_warning_threshold_mb = 100;
 
     public Integer max_streaming_retries = 3;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 39a06cb..632bf0a 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1054,6 +1054,8 @@ public class DatabaseDescriptor
         conf.compaction_throughput_mb_per_sec = value;
     }
 
+    public static int getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold_mb
* 1024 * 1024; }
+
     public static boolean getDisableSTCSInL0()
     {
         return Boolean.getBoolean("cassandra.disable_stcs_in_l0");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/492c65a6/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 30b55a0..c3caa0f 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -137,7 +137,9 @@ public class BigTableWriter extends SSTableWriter
             if (entry == null)
                 return null;
             long endPosition = dataFile.getFilePointer();
-            metadataCollector.update(endPosition - startPosition, row.columnStats());
+            long rowSize = endPosition - startPosition;
+            maybeLogLargePartitionWarning(row.key, rowSize);
+            metadataCollector.update(rowSize, row.columnStats());
             afterAppend(row.key, endPosition, entry);
             return entry;
         }
@@ -169,9 +171,20 @@ public class BigTableWriter extends SSTableWriter
         {
             throw new FSWriteError(e, dataFile.getPath());
         }
+        long rowSize = endPosition - startPosition;
+        maybeLogLargePartitionWarning(decoratedKey, rowSize);
         metadataCollector.update(endPosition - startPosition, cf.getColumnStats());
     }
 
+    private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
+    {
+        if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
+        {
+            String keyString = metadata.getKeyValidator().getString(key.getKey());
+            logger.warn("Compacting large partition {}/{}:{} ({} bytes)", metadata.ksName,
metadata.cfName, keyString, rowSize);
+        }
+    }
+
     private static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey
key, DataOutputPlus out) throws IOException
     {
         assert cf.hasColumns() || cf.isMarkedForDelete();


Mime
View raw message