cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r963159 - in /cassandra/trunk: conf/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/ src/java/org/apache/cassandra/tools/ test/ test/co...
Date Sun, 11 Jul 2010 23:07:51 GMT
Author: jbellis
Date: Sun Jul 11 23:07:50 2010
New Revision: 963159

URL: http://svn.apache.org/viewvc?rev=963159&view=rev
Log:
avoid double-copy of flushed data by serializing directly to output file.  patch by jbellis;
reviewed by mdennis for CASSANDRA-270

Modified:
    cassandra/trunk/conf/cassandra.yaml
    cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
    cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java
    cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
    cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
    cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
    cassandra/trunk/test/cassandra.in.sh
    cassandra/trunk/test/conf/cassandra.yaml
    cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
    cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java

Modified: cassandra/trunk/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/conf/cassandra.yaml?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/conf/cassandra.yaml (original)
+++ cassandra/trunk/conf/cassandra.yaml Sun Jul 11 23:07:50 2010
@@ -95,14 +95,13 @@ memtable_flush_after_mins: 60
 memtable_throughput_in_mb: 64
 # Number of objects in millions in the memtable before it is flushed
 memtable_operations_in_millions: 0.3
-# Buffer size to use when flushing !memtables to disk.
-flush_data_buffer_size_in_mb: 32
 # Increase (decrease) the index buffer size relative to the data
 # buffer if you have few (many) columns per key.
 flush_index_buffer_size_in_mb: 8
 
 column_index_size_in_kb: 64
-in_memory_compaction_limit_in_mb: 128
+
+in_memory_compaction_limit_in_mb: 64
 
 # commit log
 commitlog_directory: /var/lib/cassandra/commitlog

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Config.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Config.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Config.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Config.java Sun Jul 11 23:07:50 2010
@@ -36,9 +36,6 @@ public class Config {
     
     public Integer memtable_flush_writers = null; // will get set to the length of data dirs
in DatabaseDescriptor
     
-    public Double flush_data_buffer_size_in_mb = new Double(32);
-    public Double flush_index_buffer_size_in_mb = new Double(8);
-    
     public Integer sliced_buffer_size_in_kb = 64;
     
     public Integer storage_port = 7000;

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/Converter.java Sun Jul 11 23:07:50
2010
@@ -41,7 +41,7 @@ public class Converter {
             int size = tablesxml.getLength();
             for ( int i = 0; i < size; ++i )
             {
-                String value = null;
+                String value;
                 Keyspace ks = new Keyspace();
                 Node table = tablesxml.item(i);
                 /* parsing out the table ksName */
@@ -158,18 +158,6 @@ public class Converter {
                 conf.concurrent_writes = Integer.parseInt(rawWriters);
             }
             
-            String rawFlushData = xmlUtils.getNodeValue("/Storage/FlushDataBufferSizeInMB");
-            if (rawFlushData != null)
-            {
-                conf.flush_data_buffer_size_in_mb = Double.parseDouble(rawFlushData);
-            }
-            
-            String rawFlushIndex = xmlUtils.getNodeValue("/Storage/FlushIndexBufferSizeInMB");
-            if (rawFlushIndex != null)
-            {
-                conf.flush_index_buffer_size_in_mb = Double.parseDouble(rawFlushIndex);
-            }
-
             String rawSlicedBuffer = xmlUtils.getNodeValue("/Storage/SlicedBufferSizeInKB");
             if (rawSlicedBuffer != null)
             {
@@ -287,7 +275,7 @@ public class Converter {
     {
         try
         {
-            String configname = null;
+            String configname;
             ClassLoader loader = Converter.class.getClassLoader();
             URL scpurl = loader.getResource(PREVIOUS_CONF_FILE);
             if (scpurl == null)

Modified: cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java Sun Jul 11
23:07:50 2010
@@ -935,7 +935,7 @@ public class DatabaseDescriptor
             return conf.memtable_flush_writers;
     }
 
-    public static long getInMemoryCompactionLimit()
+    public static int getInMemoryCompactionLimit()
     {
         return conf.in_memory_compaction_limit_in_mb * 1024 * 1024;
     }
@@ -1126,16 +1126,6 @@ public class DatabaseDescriptor
         return indexAccessMode;
     }
 
-    public static double getFlushDataBufferSizeInMB()
-    {
-        return conf.flush_data_buffer_size_in_mb;
-    }
-
-    public static double getFlushIndexBufferSizeInMB()
-    {
-        return conf.flush_index_buffer_size_in_mb;
-    }
-
     public static int getIndexedReadBufferSizeInKB()
     {
         return conf.column_index_size_in_kb;

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/Memtable.java Sun Jul 11 23:07:50 2010
@@ -150,15 +150,8 @@ public class Memtable implements Compara
         logger.info("Writing " + this);
         SSTableWriter writer = new SSTableWriter(cfs.getFlushPath(), columnFamilies.size(),
partitioner);
 
-        DataOutputBuffer buffer = new DataOutputBuffer();
         for (Map.Entry<DecoratedKey, ColumnFamily> entry : columnFamilies.entrySet())
-        {
-            buffer.reset();
-            /* serialize the cf with column indexes */
-            ColumnFamily.serializer().serializeWithIndexes(entry.getValue(), buffer);
-            /* Now write the key and value to disk */
-            writer.append(entry.getKey(), buffer);
-        }
+            writer.append(entry.getKey(), entry.getValue());
 
         SSTableReader ssTable = writer.closeAndOpenReader();
         logger.info("Completed flushing " + ssTable.getFilename());

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Sun Jul 11
23:07:50 2010
@@ -43,11 +43,11 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamily;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.AbstractCompactedRow;
 import org.apache.cassandra.io.util.BufferedRandomAccessFile;
-import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.io.util.SegmentedFile;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.BloomFilter;
@@ -67,7 +67,7 @@ public class SSTableWriter extends SSTab
         super(filename, partitioner);
         iwriter = new IndexWriter(desc, partitioner, keyCount);
         dbuilder = SegmentedFile.getBuilder();
-        dataFile = new BufferedRandomAccessFile(getFilename(), "rw", (int)(DatabaseDescriptor.getFlushDataBufferSizeInMB()
* 1024 * 1024));
+        dataFile = new BufferedRandomAccessFile(getFilename(), "rw", DatabaseDescriptor.getInMemoryCompactionLimit());
     }
 
     private long beforeAppend(DecoratedKey decoratedKey) throws IOException
@@ -104,16 +104,22 @@ public class SSTableWriter extends SSTab
         afterAppend(row.key, currentPosition);
     }
 
-    // TODO make this take a DataOutputStream and wrap the byte[] version to combine them
-    public void append(DecoratedKey decoratedKey, DataOutputBuffer buffer) throws IOException
+    public void append(DecoratedKey decoratedKey, ColumnFamily cf) throws IOException
     {
-        long currentPosition = beforeAppend(decoratedKey);
+        long startPosition = beforeAppend(decoratedKey);
         FBUtilities.writeShortByteArray(partitioner.convertToDiskFormat(decoratedKey), dataFile);
-        int length = buffer.getLength();
-        assert length > 0;
-        dataFile.writeLong(length);
-        dataFile.write(buffer.getData(), 0, length);
-        afterAppend(decoratedKey, currentPosition);
+        // write placeholder for the row size, since we don't know it yet
+        long sizePosition = dataFile.getFilePointer();
+        dataFile.writeLong(-1);
+        // write out row data
+        ColumnFamily.serializer().serializeWithIndexes(cf, dataFile);
+        // seek back and write the row size (not including the size Long itself)
+        long endPosition = dataFile.getFilePointer();
+        dataFile.seek(sizePosition);
+        dataFile.writeLong(endPosition - (sizePosition + 8));
+        // finally, reset for next row
+        dataFile.seek(endPosition);
+        afterAppend(decoratedKey, startPosition);
     }
 
     public void append(DecoratedKey decoratedKey, byte[] value) throws IOException
@@ -209,7 +215,7 @@ public class SSTableWriter extends SSTab
         ffile.delete();
 
         // open the data file for input, and an IndexWriter for output
-        BufferedRandomAccessFile dfile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_DATA),
"r", (int)(DatabaseDescriptor.getFlushDataBufferSizeInMB() * 1024 * 1024));
+        BufferedRandomAccessFile dfile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_DATA),
"r", 8 * 1024 * 1024);
         IndexWriter iwriter;
         long estimatedRows;
         try
@@ -285,8 +291,7 @@ public class SSTableWriter extends SSTab
         {
             this.desc = desc;
             this.partitioner = part;
-            int bufferbytes = (int)(DatabaseDescriptor.getFlushIndexBufferSizeInMB() * 1024
* 1024);
-            indexFile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_INDEX),
"rw", bufferbytes);
+            indexFile = new BufferedRandomAccessFile(desc.filenameFor(SSTable.COMPONENT_INDEX),
"rw", 8 * 1024 * 1024);
             builder = SegmentedFile.getBuilder();
             summary = new IndexSummary();
             bf = BloomFilter.getFilter(keyCount, 15);

Modified: cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java Sun
Jul 11 23:07:50 2010
@@ -40,7 +40,6 @@ public class BufferedRandomAccessFile ex
 {
     static final int LogBuffSz_ = 16; // 64K buffer
     public static final int BuffSz_ = (1 << LogBuffSz_);
-    static final long BuffMask_ = ~(((long) BuffSz_) - 1L);
 
     private String path_;
     
@@ -84,22 +83,22 @@ public class BufferedRandomAccessFile ex
     *
     * V3. Any (possibly) unflushed characters are stored in "f.buff":
     *
-    * (forall i in [f.lo, f.curr): c(f)[i] == f.buff[i - f.lo])
+    * (forall i in [f.lo, f.hi): c(f)[i] == f.buff[i - f.lo])
     *
     * V4. For all characters not covered by V3, c(f) and disk(f) agree:
     *
-    * (forall i in [f.lo, len(f)): i not in [f.lo, f.curr) => c(f)[i] ==
+    * (forall i in [f.lo, len(f)): i not in [f.lo, f.hi) => c(f)[i] ==
     * disk(f)[i])
     *
     * V5. "f.dirty" is true iff the buffer contains bytes that should be
     * flushed to the file; by V3 and V4, only part of the buffer can be dirty.
     *
-    * f.dirty == (exists i in [f.lo, f.curr): c(f)[i] != f.buff[i - f.lo])
+    * f.dirty == (exists i in [f.lo, f.hi): c(f)[i] != f.buff[i - f.lo])
     *
     * V6. this.maxHi == this.lo + this.buff.length
     *
     * Note that "f.buff" can be "null" in a valid file, since the range of
-    * characters in V3 is empty when "f.lo == f.curr".
+    * characters in V3 is empty when "f.lo == f.hi".
     *
     * A file is said to be *ready* if the buffer contains the current position,
     * i.e., when:
@@ -189,9 +188,9 @@ public class BufferedRandomAccessFile ex
         {
             if (this.diskPos_ != this.lo_)
                 super.seek(this.lo_);
-            int len = (int) (this.curr_ - this.lo_);
+            int len = (int) (this.hi_ - this.lo_);
             super.write(this.buff_, 0, len);
-            this.diskPos_ = this.curr_;             
+            this.diskPos_ = this.hi_;
             this.dirty_ = false;
         }
     }
@@ -203,60 +202,43 @@ public class BufferedRandomAccessFile ex
      */
     private int fillBuffer() throws IOException
     {
-        int cnt = 0;
-        int rem = this.buff_.length;
-        while (rem > 0)
+        int count = 0;
+        int remainder = this.buff_.length;
+        while (remainder > 0)
         {
-            int n = super.read(this.buff_, cnt, rem);
+            int n = super.read(this.buff_, count, remainder);
             if (n < 0)
                 break;
-            cnt += n;
-            rem -= n;
+            count += n;
+            remainder -= n;
         }
-        if ( (cnt < 0) && (this.hitEOF_ = (cnt < this.buff_.length)) )
-        {
-            // make sure buffer that wasn't read is initialized with -1
-            Arrays.fill(this.buff_, cnt, this.buff_.length, (byte) 0xff);
-        }
-        this.diskPos_ += cnt;
-        return cnt;
+        this.hitEOF_ = (count < this.buff_.length);
+        this.diskPos_ += count;
+        return count;
     }
     
+    public void seek(long pos) throws IOException
+    {
+        this.curr_ = pos;
+    }
+
     /*
-     * This method positions <code>this.curr</code> at position <code>pos</code>.
-     * If <code>pos</code> does not fall in the current buffer, it flushes the
-     * current buffer and loads the correct one.<p>
-     * 
      * On exit from this routine <code>this.curr == this.hi</code> iff <code>pos</code>
      * is at or past the end-of-file, which can only happen if the file was
      * opened in read-only mode.
      */
-    public void seek(long pos) throws IOException
+    private void reBuffer() throws IOException
     {
-        if (pos >= this.hi_ || pos < this.lo_)
+        this.flushBuffer();
+        this.lo_ = this.curr_;
+        this.maxHi_ = this.lo_ + (long) this.buff_.length;
+        if (this.diskPos_ != this.lo_)
         {
-            // seeking outside of current buffer -- flush and read             
-            this.flushBuffer();
-            this.lo_ = pos & BuffMask_; // start at BuffSz boundary
-            this.maxHi_ = this.lo_ + (long) this.buff_.length;
-            if (this.diskPos_ != this.lo_)
-            {
-                super.seek(this.lo_);
-                this.diskPos_ = this.lo_;
-            }
-            int n = this.fillBuffer();
-            this.hi_ = this.lo_ + (long) n;
+            super.seek(this.lo_);
+            this.diskPos_ = this.lo_;
         }
-        else
-        {
-            // seeking inside current buffer -- no read required
-            if (pos < this.curr_)
-            {
-                // if seeking backwards, we must flush to maintain V4
-                this.flushBuffer();
-            }
-        }
-        this.curr_ = pos;
+        int n = this.fillBuffer();
+        this.hi_ = this.lo_ + (long) n;
     }
 
     public long getFilePointer()
@@ -280,16 +262,10 @@ public class BufferedRandomAccessFile ex
 
     public int read() throws IOException
     {
-        if (this.curr_ >= this.hi_)
+        if (this.lo_ > this.curr_ || this.curr_ >= this.hi_)
         {
-            // test for EOF
-            // if (this.hi < this.maxHi) return -1;
-            if (this.hitEOF_)
-                return -1;
-            
-            // slow path -- read another buffer
-            this.seek(this.curr_);
-            if (this.curr_ == this.hi_)
+            this.reBuffer();
+            if (this.curr_ == this.hi_ && this.hitEOF_)
                 return -1;
         }
         byte res = this.buff_[(int) (this.curr_ - this.lo_)];
@@ -304,16 +280,10 @@ public class BufferedRandomAccessFile ex
     
     public int read(byte[] b, int off, int len) throws IOException
     {
-        if (this.curr_ >= this.hi_)
+        if (this.lo_ > this.curr_ || this.curr_ >= this.hi_)
         {
-            // test for EOF
-            // if (this.hi < this.maxHi) return -1;
-            if (this.hitEOF_)
-                return -1;
-            
-            // slow path -- read another buffer
-            this.seek(this.curr_);
-            if (this.curr_ == this.hi_)
+            this.reBuffer();
+            if (this.curr_ == this.hi_ && this.hitEOF_)
                 return -1;
         }
         len = Math.min(len, (int) (this.hi_ - this.curr_));
@@ -325,26 +295,14 @@ public class BufferedRandomAccessFile ex
     
     public void write(int b) throws IOException
     {
-        if (this.curr_ >= this.hi_)
+        if (this.lo_ > this.curr_ || this.curr_ > this.hi_ || this.curr_ >= maxHi_)
         {
-            if (this.hitEOF_ && this.hi_ < this.maxHi_)
-            {
-                // at EOF -- bump "hi"
-                this.hi_++;
-            }
-            else
-            {
-                // slow path -- write current buffer; read next one
-                this.seek(this.curr_);
-                if (this.curr_ == this.hi_)
-                {
-                    // appending to EOF -- bump "hi"
-                    this.hi_++;
-                }
-            }
+            this.reBuffer();
         }
         this.buff_[(int) (this.curr_ - this.lo_)] = (byte) b;
         this.curr_++;
+        if (this.curr_ > this.hi_)
+            this.hi_ = this.curr_;
         this.dirty_ = true;
         syncNeeded_ = true;
     }
@@ -368,32 +326,20 @@ public class BufferedRandomAccessFile ex
     
     /*
      * Write at most "len" bytes to "b" starting at position "off", and return
-     * the number of bytes written.
+     * the number of bytes written. caller is responsible for setting dirty, syncNeeded.
      */
     private int writeAtMost(byte[] b, int off, int len) throws IOException
-    {        
-        if (this.curr_ >= this.hi_)
+    {
+        if (this.lo_ > this.curr_ || this.curr_ > this.hi_ || this.curr_ >= maxHi_)
         {
-            if (this.hitEOF_ && this.hi_ < this.maxHi_)
-            {
-                // at EOF -- bump "hi"
-                this.hi_ = this.maxHi_;
-            }
-            else
-            {                                
-                // slow path -- write current buffer; read next one                
-                this.seek(this.curr_);
-                if (this.curr_ == this.hi_)
-                {
-                    // appending to EOF -- bump "hi"
-                    this.hi_ = this.maxHi_;
-                }
-            }
+            this.reBuffer();
         }
-        len = Math.min(len, (int) (this.hi_ - this.curr_));
+        len = Math.min(len, (int) (this.maxHi_ - this.curr_));
         int buffOff = (int) (this.curr_ - this.lo_);
         System.arraycopy(b, off, this.buff_, buffOff, len);
         this.curr_ += len;
+        if (this.curr_ > this.hi_)
+            this.hi_ = this.curr_;
         return len;
     }
 

Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Sun Jul 11 23:07:50
2010
@@ -154,8 +154,7 @@ public class SSTableImport
         ColumnFamily cfamily = ColumnFamily.create(keyspace, cf);
         ColumnFamilyType cfType = cfamily.getColumnFamilyType();    // Super or Standard
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
-        DataOutputBuffer dob = new DataOutputBuffer();
-        
+
         try
         {
             JSONObject json = (JSONObject)JSONValue.parseWithException(new FileReader(jsonFile));
@@ -174,9 +173,7 @@ public class SSTableImport
                 else
                     addToStandardCF((JSONArray)json.get(rowKey.getValue()), cfamily);
                            
-                ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-                writer.append(rowKey.getKey(), dob);
-                dob.reset();
+                writer.append(rowKey.getKey(), cfamily);
                 cfamily.clear();
             }
             

Modified: cassandra/trunk/test/cassandra.in.sh
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/cassandra.in.sh?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/cassandra.in.sh (original)
+++ cassandra/trunk/test/cassandra.in.sh Sun Jul 11 23:07:50 2010
@@ -38,6 +38,7 @@ JVM_OPTS=" \
         -Xrunjdwp:transport=dt_socket,server=y,address=8898,suspend=n \
         -Xms128M \
         -Xmx1G \
+        -Xss128k \
         -XX:SurvivorRatio=8 \
         -XX:TargetSurvivorRatio=90 \
         -XX:+AggressiveOpts \

Modified: cassandra/trunk/test/conf/cassandra.yaml
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/conf/cassandra.yaml?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/conf/cassandra.yaml (original)
+++ cassandra/trunk/test/conf/cassandra.yaml Sun Jul 11 23:07:50 2010
@@ -1,6 +1,5 @@
 cluster_name: Test Cluster
-flush_data_buffer_size_in_mb: 1
-flush_index_buffer_size_in_mb: 8
+in_memory_compaction_limit_in_mb: 1
 commitlog_sync: batch
 commitlog_sync_batch_window_in_ms: 1.0
 partitioner: org.apache.cassandra.dht.CollatingOrderPreservingPartitioner

Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java Sun Jul 11
23:07:50 2010
@@ -85,28 +85,16 @@ public class SSTableUtils
 
     public static SSTableReader writeRawSSTable(String tablename, String cfname, Map<byte[],
byte[]> entries) throws IOException
     {
-        return writeRawSSTable(null, tablename, cfname, entries);
-    }
-
-    public static SSTableReader writeRawSSTable(File datafile, String tablename, String cfname,
Map<byte[], byte[]> entries) throws IOException
-    {
-        boolean temporary = false;
-        if (datafile == null)
-        {
-            datafile = tempSSTableFile(tablename, cfname);
-            temporary = true;
-        }
+        File datafile = tempSSTableFile(tablename, cfname);
         SSTableWriter writer = new SSTableWriter(datafile.getAbsolutePath(), entries.size(),
StorageService.getPartitioner());
         SortedMap<DecoratedKey, byte[]> sortedEntries = new TreeMap<DecoratedKey,
byte[]>();
         for (Map.Entry<byte[], byte[]> entry : entries.entrySet())
             sortedEntries.put(writer.partitioner.decorateKey(entry.getKey()), entry.getValue());
         for (Map.Entry<DecoratedKey, byte[]> entry : sortedEntries.entrySet())
             writer.append(entry.getKey(), entry.getValue());
-        if (temporary)
-        {
-            new File(writer.indexFilename()).deleteOnExit();
-            new File(writer.filterFilename()).deleteOnExit();
-        }
+        new File(writer.indexFilename()).deleteOnExit();
+        new File(writer.filterFilename()).deleteOnExit();
         return writer.closeAndOpenReader();
     }
+
 }

Modified: cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java?rev=963159&r1=963158&r2=963159&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/tools/SSTableExportTest.java Sun Jul 11
23:07:50 2010
@@ -60,21 +60,16 @@ public class SSTableExportTest extends S
         File tempSS = tempSSTableFile("Keyspace1", "Standard1");
         ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
-        DataOutputBuffer dob = new DataOutputBuffer();
         SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
         
         // Add rowA
         cfamily.addColumn(new QueryPath("Standard1", null, "colA".getBytes()), "valA".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowA"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowA"), cfamily);
         cfamily.clear();
         
         // Add rowB
         cfamily.addColumn(new QueryPath("Standard1", null, "colB".getBytes()), "valB".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowB"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowB"), cfamily);
         cfamily.clear();
      
         writer.closeAndOpenReader();
@@ -98,28 +93,21 @@ public class SSTableExportTest extends S
         File tempSS = tempSSTableFile("Keyspace1", "Standard1");
         ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
-        DataOutputBuffer dob = new DataOutputBuffer();
         SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
         
         // Add rowA
         cfamily.addColumn(new QueryPath("Standard1", null, "colA".getBytes()), "valA".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowA"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowA"), cfamily);
         cfamily.clear();
         
         // Add rowB
         cfamily.addColumn(new QueryPath("Standard1", null, "colB".getBytes()), "valB".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowB"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowB"), cfamily);
         cfamily.clear();
 
         // Add rowExclude
         cfamily.addColumn(new QueryPath("Standard1", null, "colX".getBytes()), "valX".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowExclude"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowExclude"), cfamily);
         cfamily.clear();
 
         SSTableReader reader = writer.closeAndOpenReader();
@@ -148,28 +136,21 @@ public class SSTableExportTest extends S
         File tempSS = tempSSTableFile("Keyspace1", "Super4");
         ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Super4");
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
-        DataOutputBuffer dob = new DataOutputBuffer();
         SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
         
         // Add rowA
         cfamily.addColumn(new QueryPath("Super4", "superA".getBytes(), "colA".getBytes()),
"valA".getBytes(), new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowA"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowA"), cfamily);
         cfamily.clear();
         
         // Add rowB
         cfamily.addColumn(new QueryPath("Super4", "superB".getBytes(), "colB".getBytes()),
"valB".getBytes(), new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowB"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowB"), cfamily);
         cfamily.clear();
 
         // Add rowExclude
         cfamily.addColumn(new QueryPath("Super4", "superX".getBytes(), "colX".getBytes()),
"valX".getBytes(), new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowExclude"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowExclude"), cfamily);
         cfamily.clear();
 
         SSTableReader reader = writer.closeAndOpenReader();
@@ -196,21 +177,16 @@ public class SSTableExportTest extends S
         File tempSS = tempSSTableFile("Keyspace1", "Standard1");
         ColumnFamily cfamily = ColumnFamily.create("Keyspace1", "Standard1");
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();
-        DataOutputBuffer dob = new DataOutputBuffer();
         SSTableWriter writer = new SSTableWriter(tempSS.getPath(), 2, partitioner);
         
         // Add rowA
         cfamily.addColumn(new QueryPath("Standard1", null, "name".getBytes()), "val".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowA"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowA"), cfamily);
         cfamily.clear();
 
         // Add rowExclude
         cfamily.addColumn(new QueryPath("Standard1", null, "name".getBytes()), "val".getBytes(),
new TimestampClock(1));
-        ColumnFamily.serializer().serializeWithIndexes(cfamily, dob);
-        writer.append(Util.dk("rowExclude"), dob);
-        dob.reset();
+        writer.append(Util.dk("rowExclude"), cfamily);
         cfamily.clear();
 
         SSTableReader reader = writer.closeAndOpenReader();



Mime
View raw message