cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [4/8] git commit: memtable_cleanup_threshold and memtable_flush_writers defaults patch by jbellis; reviewed by bes for CASSANDRA-7551
Date Tue, 15 Jul 2014 20:15:27 GMT
memtable_cleanup_threshold and memtable_flush_writers defaults
patch by jbellis; reviewed by bes for CASSANDRA-7551


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3adf5948
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3adf5948
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3adf5948

Branch: refs/heads/trunk
Commit: 3adf59486a4e55f409fcc62a4c942a070b98dd8f
Parents: bd0eafd
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Tue Jul 15 15:14:50 2014 -0500
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Tue Jul 15 15:14:56 2014 -0500

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 conf/cassandra.yaml                             | 23 +++++++++++++++-----
 .../org/apache/cassandra/config/Config.java     |  6 ++---
 .../cassandra/config/DatabaseDescriptor.java    |  6 +++++
 4 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e6ded2a..b20c465 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,6 @@
 2.1.0-final
+ * Updated memtable_cleanup_threshold and memtable_flush_writers defaults 
+   (CASSANDRA-7551)
  * (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
  * Anti-compaction proceeds if any part of the repair failed (CASANDRA-7521)
  * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index f1e5576..4ce69fe 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -282,8 +282,13 @@ concurrent_counter_writes: 32
 # memtable_total_space_in_mb: 2048
 
 # Ratio of occupied non-flushing memtable size to total permitted size
-# that will trigger a flush of the largest memtable.
-memtable_cleanup_threshold: 0.4
+# that will trigger a flush of the largest memtable.  Lager mct will
+# mean larger flushes and hence less compaction, but also less concurrent
+# flush activity which can make it difficult to keep your disks fed
+# under heavy write load.
+#
+# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1)
+#memtable_cleanup_threshold: 0.11
 
 # Specify the way Cassandra allocates and manages memtable memory.
 # Options are:
@@ -304,9 +309,14 @@ memtable_allocation_type: heap_buffers
 
 # This sets the amount of memtable flush writer threads.  These will
 # be blocked by disk io, and each one will hold a memtable in memory
-# while blocked. If your flush directory is backed by SSD, you may
-# want to increase this.
-memtable_flush_writers: 2
+# while blocked. 
+#
+# memtable_flush_writers defaults to the smaller of (number of disks,
+# number of cores), with a minimum of 2 and a maximum of 8.
+# 
+# If your data directories are backed by SSD, you should increase this
+# to the number of cores.
+#memtable_flush_writers: 8
 
 # A fixed memory pool size in MB for for SSTable index summaries. If left
 # empty, this will default to 5% of the heap size. If the memory usage of
@@ -503,6 +513,9 @@ batch_size_warn_threshold_in_kb: 5
 #
 # concurrent_compactors defaults to the smaller of (number of disks,
 # number of cores), with a minimum of 2 and a maximum of 8.
+# 
+# If your data directories are backed by SSD, you should increase this
+# to the number of cores.
 #concurrent_compactors: 1
 
 # Throttles compaction to the given total throughput across the entire

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 4d4a95b..2438e09 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -90,12 +90,10 @@ public class Config
     @Deprecated
     public Integer concurrent_replicates = null;
 
-    // we don't want a lot of contention, but we also don't want to starve all other tables
-    // if a big one flushes. OS buffering should be able to minimize contention with 2 threads.
-    public int memtable_flush_writers = 2;
+    public Integer memtable_flush_writers = null;
     public Integer memtable_heap_space_in_mb;
     public Integer memtable_offheap_space_in_mb;
-    public float memtable_cleanup_threshold = 0.4f;
+    public Double memtable_cleanup_threshold = null;
 
     public Integer storage_port = 7000;
     public Integer ssl_storage_port = 7001;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3adf5948/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 5a2f26f..6730cb5 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -483,6 +483,12 @@ public class DatabaseDescriptor
         if (conf.commitlog_directory.equals(conf.saved_caches_directory))
             throw new ConfigurationException("saved_caches_directory must not be the same
as the commitlog_directory");
 
+        if (conf.memtable_flush_writers == null)
+            conf.memtable_flush_writers = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(),
conf.data_file_directories.length)));
+
+        if (conf.memtable_cleanup_threshold == null)
+            conf.memtable_cleanup_threshold = 1.0 / (1 + conf.memtable_flush_writers);
+
         if (conf.concurrent_compactors == null)
             conf.concurrent_compactors = Math.min(8, Math.max(2, Math.min(FBUtilities.getAvailableProcessors(),
conf.data_file_directories.length)));
 


Mime
View raw message