cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r898175 - in /incubator/cassandra/trunk/src/java/org/apache/cassandra: db/ColumnFamilyStore.java db/Memtable.java db/Table.java db/filter/NamesQueryFilter.java db/filter/QueryFilter.java db/filter/SliceQueryFilter.java io/SSTableReader.java
Date Tue, 12 Jan 2010 04:03:19 GMT
Author: jbellis
Date: Tue Jan 12 04:03:19 2010
New Revision: 898175

URL: http://svn.apache.org/viewvc?rev=898175&view=rev
Log:
basic row caching.
patch by jbellis; reviewed by goffinet for CASSANDRA-678

Modified:
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/QueryFilter.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Tue
Jan 12 04:03:19 2010
@@ -30,6 +30,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import com.reardencommerce.kernel.collections.shared.evictable.ConcurrentLinkedHashMap;
 import org.apache.cassandra.service.SliceRange;
 import org.apache.log4j.Logger;
 
@@ -105,6 +106,8 @@
     /* active memtable associated with this ColumnFamilyStore. */
     private Memtable memtable_;
 
+    private ConcurrentLinkedHashMap<String, ColumnFamily> rowCache;
+
     // TODO binarymemtable ops are not threadsafe (do they need to be?)
     private AtomicReference<BinaryMemtable> binaryMemtable_;
 
@@ -124,6 +127,8 @@
         fileIndexGenerator_.set(indexValue);
         memtable_ = new Memtable(table_, columnFamily_);
         binaryMemtable_ = new AtomicReference<BinaryMemtable>(new BinaryMemtable(table_,
columnFamily_));
+        int cacheSize = SSTableReader.estimatedKeys(columnFamilyName);
+        rowCache = ConcurrentLinkedHashMap.create(ConcurrentLinkedHashMap.EvictionPolicy.SECOND_CHANCE,
cacheSize);
     }
 
     public static ColumnFamilyStore getColumnFamilyStore(String table, String columnFamily)
throws IOException
@@ -780,6 +785,19 @@
         return getColumnFamily(filter, CompactionManager.getDefaultGCBefore());
     }
 
+    private ColumnFamily getCachedRow(QueryFilter filter) throws IOException
+    {
+        ColumnFamily cached;
+        if ((cached = rowCache.get(filter.key)) == null)
+        {
+            cached = getTopLevelColumns(new IdentityQueryFilter(filter.key, new QueryPath(columnFamily_)),
Integer.MIN_VALUE);
+            if (cached == null)
+                return null;
+            rowCache.put(filter.key, cached);
+        }
+        return cached;
+    }
+
     /**
      * get a list of columns starting from a given column, in a specified order.
      * only the latest version of a column is returned.
@@ -792,19 +810,37 @@
         long start = System.currentTimeMillis();
         try
         {
-            // if we are querying subcolumns of a supercolumn, fetch the supercolumn with
NQF, then filter in-memory.
             if (filter.path.superColumnName == null)
             {
-                return removeDeleted(getTopLevelColumns(filter, gcBefore), gcBefore);
+                if (rowCache == null)
+                    return removeDeleted(getTopLevelColumns(filter, gcBefore), gcBefore);
+
+                ColumnFamily cached = getCachedRow(filter);
+                ColumnIterator ci = filter.getMemColumnIterator(memtable_, cached, getComparator());
// TODO passing memtable here is confusing since it's almost entirely unused
+                ColumnFamily returnCF = ci.getColumnFamily();
+                filter.collectCollatedColumns(returnCF, ci, gcBefore);
+                return removeDeleted(returnCF);
             }
 
-            QueryFilter nameFilter = new NamesQueryFilter(filter.key, new QueryPath(columnFamily_),
filter.path.superColumnName);
-            ColumnFamily cf = getTopLevelColumns(nameFilter, gcBefore);
-            if (cf == null || cf.getColumnCount() == 0)
-                return cf;
+            // we are querying subcolumns of a supercolumn: fetch the supercolumn with NQF,
then filter in-memory.
+            ColumnFamily cf;
+            SuperColumn sc;
+            if (rowCache == null)
+            {
+                QueryFilter nameFilter = new NamesQueryFilter(filter.key, new QueryPath(columnFamily_),
filter.path.superColumnName);
+                cf = getTopLevelColumns(nameFilter, gcBefore);
+                if (cf == null || cf.getColumnCount() == 0)
+                    return cf;
 
-            assert cf.getSortedColumns().size() == 1;
-            SuperColumn sc = (SuperColumn)cf.getSortedColumns().iterator().next();
+                assert cf.getSortedColumns().size() == 1;
+                sc = (SuperColumn)cf.getSortedColumns().iterator().next();
+            }
+            else
+            {
+                cf = getCachedRow(filter);
+                sc = (SuperColumn)cf.getColumn(filter.path.superColumnName);
+            }
+            
             SuperColumn scFiltered = filter.filterSuperColumn(sc, gcBefore);
             ColumnFamily cfFiltered = cf.cloneMeShallow();
             cfFiltered.addColumn(scFiltered);
@@ -1222,6 +1258,11 @@
         return ssTables_.size();
     }
 
+    public void invalidate(String key)
+    {
+        rowCache.remove(key);
+    }
+
     /**
      * for testing.  no effort is made to clear historical memtables.
      */

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Tue Jan 12 04:03:19
2010
@@ -199,9 +199,8 @@
     /**
      * obtain an iterator of columns in this memtable in the specified order starting from
a given column.
      */
-    public ColumnIterator getSliceIterator(SliceQueryFilter filter, AbstractType typeComparator)
+    public ColumnIterator getSliceIterator(ColumnFamily cf, SliceQueryFilter filter, AbstractType
typeComparator)
     {
-        ColumnFamily cf = columnFamilies.get(partitioner.decorateKey(filter.key));
         final ColumnFamily columnFamily = cf == null ? ColumnFamily.create(table, filter.getColumnFamilyName())
: cf.cloneMeShallow();
 
         final IColumn columns[] = (cf == null ? columnFamily : cf).getSortedColumns().toArray(new
IColumn[columnFamily.getSortedColumns().size()]);
@@ -252,9 +251,8 @@
         };
     }
 
-    public ColumnIterator getNamesIterator(final NamesQueryFilter filter)
+    public ColumnIterator getNamesIterator(final ColumnFamily cf, final NamesQueryFilter
filter)
     {
-        final ColumnFamily cf = columnFamilies.get(partitioner.decorateKey(filter.key));
         final ColumnFamily columnFamily = cf == null ? ColumnFamily.create(table, filter.getColumnFamilyName())
: cf.cloneMeShallow();
         final boolean isStandard = DatabaseDescriptor.getColumnFamilyType(table, filter.getColumnFamilyName()).equals("Standard");
 
@@ -287,6 +285,11 @@
         };
     }
 
+    public ColumnFamily getColumnFamily(String key)
+    {
+        return columnFamilies.get(partitioner.decorateKey(key));
+    }
+
     void clearUnsafe()
     {
         columnFamilies.clear();

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Tue Jan 12 04:03:19
2010
@@ -415,6 +415,7 @@
     {
         HashMap<ColumnFamilyStore,Memtable> memtablesToFlush = new HashMap<ColumnFamilyStore,
Memtable>(2);
 
+        // write the mutation to the commitlog and memtables
         flusherLock.readLock().lock();
         try
         {
@@ -434,7 +435,14 @@
             flusherLock.readLock().unlock();
         }
 
-        // usually mTF will be empty and this will be a no-op
+        // invalidate cache.  2nd loop over CFs here to avoid prolonging the lock section
unnecessarily.
+        for (ColumnFamily cf : mutation.getColumnFamilies())
+        {
+            ColumnFamilyStore cfs = columnFamilyStores.get(cf.name());
+            cfs.invalidate(mutation.key());
+        }
+
+        // flush memtables that got filled up.  usually mTF will be empty and this will be
a no-op
         for (Map.Entry<ColumnFamilyStore, Memtable> entry : memtablesToFlush.entrySet())
             entry.getKey().switchMemtable(entry.getValue(), writeCommitLog);
     }

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/NamesQueryFilter.java
Tue Jan 12 04:03:19 2010
@@ -59,9 +59,9 @@
         return set;
     }
 
-    public ColumnIterator getMemColumnIterator(Memtable memtable, AbstractType comparator)
+    public ColumnIterator getMemColumnIterator(Memtable memtable, ColumnFamily cf, AbstractType
comparator)
     {
-        return memtable.getNamesIterator(this);
+        return memtable.getNamesIterator(cf, this);
     }
 
     public ColumnIterator getSSTableColumnIterator(SSTableReader sstable) throws IOException

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/QueryFilter.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/QueryFilter.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/QueryFilter.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/QueryFilter.java Tue
Jan 12 04:03:19 2010
@@ -46,7 +46,12 @@
      * returns an iterator that returns columns from the given memtable
      * matching the Filter criteria in sorted order.
      */
-    public abstract ColumnIterator getMemColumnIterator(Memtable memtable, AbstractType comparator);
+    public abstract ColumnIterator getMemColumnIterator(Memtable memtable, ColumnFamily cf,
AbstractType comparator);
+
+    public ColumnIterator getMemColumnIterator(Memtable memtable, AbstractType comparator)
+    {
+        return getMemColumnIterator(memtable, memtable.getColumnFamily(key), comparator);
+    }
 
     /**
      * returns an iterator that returns columns from the given SSTable

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java
Tue Jan 12 04:03:19 2010
@@ -53,9 +53,9 @@
         this.count = count;
     }
 
-    public ColumnIterator getMemColumnIterator(Memtable memtable, AbstractType comparator)
+    public ColumnIterator getMemColumnIterator(Memtable memtable, ColumnFamily cf, AbstractType
comparator)
     {
-        return memtable.getSliceIterator(this, comparator);
+        return memtable.getSliceIterator(cf, this, comparator);
     }
 
     public ColumnIterator getSSTableColumnIterator(SSTableReader sstable) throws IOException

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java?rev=898175&r1=898174&r2=898175&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/SSTableReader.java Tue Jan
12 04:03:19 2010
@@ -117,6 +117,17 @@
         return count;
     }
 
+    public static int estimatedKeys(String columnFamilyName)
+    {
+        int n = 0;
+        for (SSTableReader sstable : openedFiles.values())
+        {
+            if (sstable.getColumnFamilyName().equals(columnFamilyName))
+                n += sstable.getIndexPositions().size() * INDEX_INTERVAL;
+        }
+        return n;
+    }
+
     /**
      * Get all indexed keys defined by the two predicates.
      * @param cfpred A Predicate defining matching column families.



Mime
View raw message