cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [1/8] cassandra git commit: Warn when an extra-large partition is compacted, CASSANDRA-9643
Date Thu, 02 Jul 2015 18:58:29 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.2 1dbbf6040 -> 492c65a67
  refs/heads/trunk 3f3fdf377 -> fae9d3bea


Warn when an extra-large partition is compacted, CASSANDRA-9643


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/921eb7e8
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/921eb7e8
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/921eb7e8

Branch: refs/heads/cassandra-2.2
Commit: 921eb7e8065221be586f63c64f9b1de582136ca5
Parents: 2595e74
Author: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Authored: Tue Jun 30 17:12:56 2015 +0800
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Thu Jul 2 11:37:50 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                        |  1 +
 conf/cassandra.yaml                                |  3 +++
 src/java/org/apache/cassandra/config/Config.java   |  1 +
 .../cassandra/config/DatabaseDescriptor.java       |  2 ++
 .../apache/cassandra/io/sstable/SSTableWriter.java | 17 +++++++++++++++--
 5 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c37ef02..78e23fb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.8
+ * Warn when an extra-large partition is compacted (CASSANDRA-9643)
  * Eliminate strong self-reference chains in sstable ref tidiers (CASSANDRA-9656)
  * Ensure StreamSession uses canonical sstable reader instances (CASSANDRA-9700) 
  * Ensure memtable book keeping is not corrupted in the event we shrink usage (CASSANDRA-9681)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 9d6d135..e4b7cbd 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -574,6 +574,9 @@ batch_size_warn_threshold_in_kb: 5
 # of compaction, including validation compaction.
 compaction_throughput_mb_per_sec: 16
 
+# Log a warning when compacting partitions larger than this value
+compaction_large_partition_warning_threshold_mb: 100
+
 # When compacting, the replacement sstable(s) can be opened before they
 # are completely written, and used in place of the prior sstables for
 # any range that has been written. This helps to smoothly transfer reads 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 201e729..31e4cd5 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -146,6 +146,7 @@ public class Config
     public Integer batch_size_warn_threshold_in_kb = 5;
     public Integer concurrent_compactors;
     public volatile Integer compaction_throughput_mb_per_sec = 16;
+    public volatile Integer compaction_large_partition_warning_threshold_mb = 100;
 
     public Integer max_streaming_retries = 3;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index a2a9e48..68dcd5c 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1103,6 +1103,8 @@ public class DatabaseDescriptor
         conf.compaction_throughput_mb_per_sec = value;
     }
 
+    public static int getCompactionLargePartitionWarningThreshold() { return conf.compaction_large_partition_warning_threshold_mb
* 1024 * 1024; }
+
     public static boolean getDisableSTCSInL0()
     {
         return Boolean.getBoolean("cassandra.disable_stcs_in_l0");

http://git-wip-us.apache.org/repos/asf/cassandra/blob/921eb7e8/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index a39c134..8e0b5f7 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -198,7 +198,9 @@ public class SSTableWriter extends SSTable
             throw new FSWriteError(e, dataFile.getPath());
         }
         long endPosition = dataFile.getFilePointer();
-        sstableMetadataCollector.update(endPosition - startPosition, row.columnStats());
+        long rowSize = endPosition - startPosition;
+        maybeLogLargePartitionWarning(row.key, rowSize);
+        sstableMetadataCollector.update(rowSize, row.columnStats());
         afterAppend(row.key, endPosition, entry);
         return entry;
     }
@@ -225,7 +227,18 @@ public class SSTableWriter extends SSTable
         {
             throw new FSWriteError(e, dataFile.getPath());
         }
-        sstableMetadataCollector.update(endPosition - startPosition, cf.getColumnStats());
+        long rowSize = endPosition - startPosition;
+        maybeLogLargePartitionWarning(decoratedKey, rowSize);
+        sstableMetadataCollector.update(rowSize, cf.getColumnStats());
+    }
+
+    private void maybeLogLargePartitionWarning(DecoratedKey key, long rowSize)
+    {
+        if (rowSize > DatabaseDescriptor.getCompactionLargePartitionWarningThreshold())
+        {
+            String keyString = metadata.getKeyValidator().getString(key.getKey());
+            logger.warn("Compacting large partition {}/{}:{} ({} bytes)", metadata.ksName,
metadata.cfName, keyString, rowSize);
+        }
     }
 
     public static RowIndexEntry rawAppend(ColumnFamily cf, long startPosition, DecoratedKey
key, DataOutputPlus out) throws IOException


Mime
View raw message