cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r821127 - in /incubator/cassandra/trunk: conf/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/service/
Date Fri, 02 Oct 2009 18:40:27 GMT
Author: jbellis
Date: Fri Oct  2 18:40:27 2009
New Revision: 821127

URL: http://svn.apache.org/viewvc?rev=821127&view=rev
Log:
add MemtableFlushAfterMinutes, a global replacement for the old per-CF FlushPeriodInMinutes
setting
patch by jbellis; reviewed by junrao for CASSANDRA-463

Modified:
    incubator/cassandra/trunk/conf/storage-conf.xml
    incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java

Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Fri Oct  2 18:40:27 2009
@@ -58,15 +58,8 @@
        ~
        ~ (To get the closest approximation to 0.3-style supercolumns, you
        ~ would use CompareWith=UTF8Type CompareSubcolumnsWith=LongType.)
-
-       ~ If FlushPeriodInMinutes is positive, it will be
-       ~ flushed to disk with that period even if the memtable has not
-       ~ exceeded the size or count thresholds, so that it does not
-       ~ prevent commitlog segments from being purged. 
       -->
-      <ColumnFamily CompareWith="BytesType"
-                    Name="Standard1"
-                    FlushPeriodInMinutes="60"/>
+      <ColumnFamily CompareWith="BytesType" Name="Standard1"/>
       <ColumnFamily CompareWith="UTF8Type" Name="Standard2"/>
       <ColumnFamily CompareWith="TimeUUIDType" Name="StandardByUUID1"/>
       <ColumnFamily ColumnType="Super"
@@ -243,6 +236,16 @@
    ~ setting.  Use with MemtableSizeInMB to tune memory usage.
   -->
   <MemtableObjectCountInMillions>0.1</MemtableObjectCountInMillions>
+  <!--
+   ~ The maximum time to leave a dirty memtable unflushed.
+   ~ (While any affected columnfamilies have unflushed data from a
+   ~ commit log segment, that segment cannot be deleted.)
+   ~ This needs to be large enough that it won't cause a flush storm
+   ~ of all your memtables flushing at once because none has hit
+   ~ the size or count thresholds yet.  For production, a larger
+   ~ value such as 1440 is recommended.
+  -->
+  <MemtableFlushAfterMinutes>60</MemtableFlushAfterMinutes>
 
   <!--
    ~ Unlike most systems, in Cassandra writes are faster than reads, so

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/CFMetaData.java Fri Oct
 2 18:40:27 2009
@@ -36,7 +36,6 @@
     public String n_columnKey;
     public String n_columnValue;
     public String n_columnTimestamp;
-    public int    flushPeriodInMinutes = 0; // flush interval, if <=0, no periodic flusher
is scheduled
 
     // a quick and dirty pretty printer for describing the column family...
     public String pretty()
@@ -51,7 +50,6 @@
         
         desc += "Column Family Type: " + columnType + "\n" +
                 "Columns Sorted By: " + comparator + "\n";
-        desc += "flush period: " + flushPeriodInMinutes + " minutes\n";
         return desc;
     }
 }

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Fri Oct  2 18:40:27 2009
@@ -20,7 +20,6 @@
 
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.dht.IPartitioner;
@@ -101,8 +100,8 @@
 
     /* if the size of columns or super-columns are more than this, indexing will kick in
*/
     private static int columnIndexSizeInKB_;
-    /* Number of hours to keep a memtable in memory */
-    private static int memtableLifetime_ = 6;
+    /* Number of minutes to keep a memtable in memory */
+    private static int memtableLifetimeMs_ = 60 * 60 * 1000;
     /* Size of the memtable in memory before it is dumped */
     private static int memtableSize_ = 128;
     /* Number of objects in millions in the memtable before it is dumped */
@@ -320,9 +319,9 @@
             
 
             /* Number of days to keep the memtable around w/o flushing */
-            String lifetime = xmlUtils.getNodeValue("/Storage/MemtableLifetimeInDays");
-            if ( lifetime != null )
-                memtableLifetime_ = Integer.parseInt(lifetime);
+            String lifetime = xmlUtils.getNodeValue("/Storage/MemtableFlushAfterMinutes");
+            if (lifetime != null)
+                memtableLifetimeMs_ = Integer.parseInt(lifetime) * 60 * 1000;
 
             /* Size of the memtable in memory in MB before it is dumped */
             String memtableSize = xmlUtils.getNodeValue("/Storage/MemtableSizeInMB");
@@ -471,13 +470,6 @@
                         throw new ConfigurationException("CompareSubcolumnsWith is only a
valid attribute on super columnfamilies (not regular columnfamily " + cfName + ")");
                     }
 
-                    // see if flush period is set
-                    String flushPeriodInMinutes = XMLUtils.getAttributeValue(columnFamily,
"FlushPeriodInMinutes");
-                    int flushPeriod=0;
-                    if ( flushPeriodInMinutes != null )
-                        flushPeriod = Integer.parseInt(flushPeriodInMinutes);
-
-                    
                     // Parse out user-specified logical names for the various dimensions
                     // of a the column family from the config.
                     String n_superColumnMap = xmlUtils.getNodeValue(xqlCF + "SuperColumnMap");
@@ -525,8 +517,7 @@
                         cfMetaData.n_superColumnKey = n_superColumnKey;
                         cfMetaData.n_superColumnMap = n_superColumnMap;
                     }
-                    cfMetaData.flushPeriodInMinutes = flushPeriod;
-                    
+
                     tableToCFMetaDataMap_.get(tName).put(cfName, cfMetaData);
                 }
             }
@@ -538,14 +529,12 @@
             CFMetaData data = new CFMetaData();
             data.columnType = "Standard";
             data.comparator = new UTF8Type();
-            data.flushPeriodInMinutes = 1;
             systemMetadata.put(SystemTable.LOCATION_CF, data);
 
             data = new CFMetaData();
             data.columnType = "Super";
             data.comparator = new UTF8Type();
             data.subcolumnComparator = new BytesType();
-            data.flushPeriodInMinutes = 10;
             systemMetadata.put(HintedHandOffManager.HINTS_CF, data);
             tableToCFMetaDataMap_.put(Table.SYSTEM_TABLE, systemMetadata);
 
@@ -673,9 +662,9 @@
     	return columnIndexSizeInKB_ * 1024;
     }
 
-    public static int getMemtableLifetime()
+    public static int getMemtableLifetimeMS()
     {
-      return memtableLifetime_;
+      return memtableLifetimeMs_;
     }
 
     public static String getInitialToken()
@@ -748,16 +737,6 @@
         return cfMetaData.columnType;
     }
 
-    public static int getFlushPeriod(String tableName, String columnFamilyName)
-    {
-        assert tableName != null;
-        CFMetaData cfMetaData = getCFMetaData(tableName, columnFamilyName);
-        
-        if (cfMetaData == null)
-            return 0;
-        return cfMetaData.flushPeriodInMinutes;
-    }
-
     public static List<String> getTables()
     {
         return tables_;

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Fri
Oct  2 18:40:27 2009
@@ -85,7 +85,6 @@
                                                new LinkedBlockingQueue<Runnable>(),
                                                new NamedThreadFactory("FLUSH-WRITER-POOL"));
     private static ExecutorService commitLogUpdater_ = new DebuggableThreadPoolExecutor("MEMTABLE-POST-FLUSHER");
-    private static Timer flushTimer_ = new Timer("FLUSH-TIMER");
 
     private final String table_;
     public final String columnFamily_;
@@ -223,26 +222,6 @@
         {
             HintedHandOffManager.instance().scheduleHandoffsFor(this);
         }
-
-        // schedule periodic flusher if required
-        int flushPeriodMS = DatabaseDescriptor.getFlushPeriod(table_, columnFamily_) * 60
* 1000;
-        if (flushPeriodMS > 0)
-        {
-            flushTimer_.schedule(new TimerTask()
-            {
-                public void run()
-                {
-                    try
-                    {
-                        forceFlush();
-                    }
-                    catch (IOException e)
-                    {
-                        throw new RuntimeException(e);
-                    }
-                }
-            }, flushPeriodMS, flushPeriodMS);
-        }
     }
 
     /*
@@ -417,6 +396,12 @@
         binaryMemtable_.get().put(key, buffer);
     }
 
+    public void forceFlushIfExpired() throws IOException
+    {
+        if (memtable_.isExpired())
+            forceFlush();
+    }
+
     public Future<?> forceFlush() throws IOException
     {
         if (memtable_.isClean())

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Fri Oct  2 18:40:27
2009
@@ -346,4 +346,9 @@
     {
         columnFamilies_.clear();
     }
+
+    public boolean isExpired()
+    {
+        return System.currentTimeMillis() > creationTime_ + DatabaseDescriptor.getMemtableLifetimeMS();
+    }
 }

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Fri Oct  2 18:40:27
2009
@@ -53,6 +53,8 @@
     /* we use this lock to drain updaters before calling a flush. */
     static final ReentrantReadWriteLock flusherLock_ = new ReentrantReadWriteLock(true);
 
+    private static Timer flushTimer_ = new Timer("FLUSH-TIMER");
+
     /*
      * This class represents the metadata of this Table. The metadata
      * is basically the column family name and the ID associated with
@@ -513,6 +515,26 @@
         {
             columnFamilyStores_.put(columnFamily, ColumnFamilyStore.getColumnFamilyStore(table,
columnFamily));
         }
+
+        // check 10x as often as the lifetime, so we can exceed lifetime by 10% at most
+        int checkMs = DatabaseDescriptor.getMemtableLifetimeMS() / 10;
+        flushTimer_.schedule(new TimerTask()
+        {
+            public void run()
+            {
+                for (ColumnFamilyStore cfs : columnFamilyStores_.values())
+                {
+                    try
+                    {
+                        cfs.forceFlushIfExpired();
+                    }
+                    catch (IOException e)
+                    {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+        }, checkMs, checkMs);
     }
 
     boolean isApplicationColumnFamily(String columnFamily)

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java?rev=821127&r1=821126&r2=821127&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/service/CassandraServer.java Fri
Oct  2 18:40:27 2009
@@ -572,7 +572,6 @@
             {
                 columnMap.put("CompareSubcolumnsWith", columnFamilyMetaData.subcolumnComparator.getClass().getName());
             }
-            columnMap.put("FlushPeriodInMinutes", columnFamilyMetaData.flushPeriodInMinutes
+ "");
             columnFamiliesMap.put(columnFamilyMetaData.cfName, columnMap);
         }
         return columnFamiliesMap;



Mime
View raw message