cassandra-commits mailing list archives

Author: slebresne
Date: Fri Sep  2 17:53:05 2011
New Revision: 1164657

URL: http://svn.apache.org/viewvc?rev=1164657&view=rev
Log:
Improve SSTableSimpleUnsortedWriter speed with large rows
patch by slebresne; reviewed by jebllis for CASSANDRA-3122

Modified:
    cassandra/branches/cassandra-0.8/CHANGES.txt
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java

Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1164657&r1=1164656&r2=1164657&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Fri Sep  2 17:53:05 2011
@@ -52,6 +52,7 @@
  * optionally skip log4j configuration (CASSANDRA-3061)
  * bundle sstableloader with the debian package (CASSANDRA-3113)
  * don't try to build secondary indexes when there is none (CASSANDRA-3123)
+ * improve SSTableSimpleUnsortedWriter speed for large rows (CASSANDRA-3122)
 
 
 0.8.4

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java?rev=1164657&r1=1164656&r2=1164657&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
Fri Sep  2 17:53:05 2011
@@ -94,7 +94,7 @@ public abstract class AbstractSSTableSim
             writeRow(currentKey, columnFamily);
 
         currentKey = StorageService.getPartitioner().decorateKey(key);
-        columnFamily = ColumnFamily.create(metadata);
+        columnFamily = getColumnFamily();
     }
 
     /**
@@ -163,4 +163,6 @@ public abstract class AbstractSSTableSim
     public abstract void close() throws IOException;
 
     protected abstract void writeRow(DecoratedKey key, ColumnFamily columnFamily) throws
IOException;
+
+    protected abstract ColumnFamily getColumnFamily();
 }

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java?rev=1164657&r1=1164656&r2=1164657&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
Fri Sep  2 17:53:05 2011
@@ -29,6 +29,8 @@ import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.service.StorageService;
 
+import org.apache.cassandra.utils.ByteBufferUtil;
+
 /**
  * A SSTable writer that doesn't assume rows are in sorted order.
  * This writer buffers rows in memory and then write them all in sorted order.
@@ -68,18 +70,30 @@ public class SSTableSimpleUnsortedWriter
 
     protected void writeRow(DecoratedKey key, ColumnFamily columnFamily) throws IOException
     {
-        ColumnFamily previous = keys.put(key, columnFamily);
         currentSize += key.key.remaining() + columnFamily.serializedSize() * 1.2;
 
-        // Note that if the row was existing already, our size estimation will be slightly
off
-        // since we'll be counting the key multiple times.
-        if (previous != null)
-            columnFamily.addAll(previous);
-
         if (currentSize > bufferSize)
             sync();
     }
 
+    protected ColumnFamily getColumnFamily()
+    {
+        ColumnFamily previous = keys.get(currentKey);
+        // If the CF already exist in memory, we'll just continue adding to it
+        if (previous == null)
+        {
+            previous = ColumnFamily.create(metadata);
+            keys.put(currentKey, previous);
+        }
+        else
+        {
+            // We will reuse a CF that we have counted already. But because it will be easier
to add the full size
+            // of the CF in the next writeRow call than to find out the delta, we just remove
the size until that next call
+            currentSize -= currentKey.key.remaining() + previous.serializedSize() * 1.2;
+        }
+        return previous;
+    }
+
     public void close() throws IOException
     {
         sync();

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java?rev=1164657&r1=1164656&r2=1164657&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
Fri Sep  2 17:53:05 2011
@@ -76,4 +76,9 @@ public class SSTableSimpleWriter extends
     {
         writer.append(key, columnFamily);
     }
+
+    protected ColumnFamily getColumnFamily()
+    {
+        return ColumnFamily.create(metadata);
+    }
 }