cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r893584 - in /incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra: db/ColumnFamilyStore.java io/CompactionIterator.java io/SSTableReader.java io/SSTableScanner.java tools/SSTableExport.java
Date Wed, 23 Dec 2009 17:46:06 GMT
Author: jbellis
Date: Wed Dec 23 17:45:58 2009
New Revision: 893584

URL: http://svn.apache.org/viewvc?rev=893584&view=rev
Log:
allow specifying buffer size to SSTableScanner.  patch by Stu Hood and jbellis for CASSANDRA-607

Modified:
    incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
    incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
    incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
    incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java

Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Wed Dec 23 17:45:58 2009
@@ -96,6 +96,8 @@
                                                new NamedThreadFactory("FLUSH-WRITER-POOL"));
     private static ExecutorService commitLogUpdater_ = new DebuggableThreadPoolExecutor("MEMTABLE-POST-FLUSHER");
 
+    private static final int KEY_RANGE_FILE_BUFFER_SIZE = 256 * 1024;
+
     private final String table_;
     public final String columnFamily_;
     private final boolean isSuper_;
@@ -785,7 +787,7 @@
      * @return
      * @throws IOException
      */
-    List<SSTableReader> doFileAntiCompaction(Collection<SSTableReader> sstables,
Collection<Range> ranges, InetAddress target) throws IOException
+    List<SSTableReader> doFileAntiCompaction(Collection<SSTableReader> sstables,
final Collection<Range> ranges, InetAddress target) throws IOException
     {
         logger_.info("AntiCompacting [" + StringUtils.join(sstables, ",") + "]");
         // Calculate the expected compacted filesize
@@ -1315,7 +1317,7 @@
         // sstables
         for (SSTableReader sstable : ssTables_)
         {
-            final SSTableScanner scanner = sstable.getScanner();
+            final SSTableScanner scanner = sstable.getScanner(KEY_RANGE_FILE_BUFFER_SIZE);
             scanner.seekTo(startWithDK);
             Iterator<DecoratedKey> iter = new CloseableIterator<DecoratedKey>()
             {
@@ -1434,7 +1436,7 @@
         // sstables
         for (SSTableReader sstable : ssTables_)
         {
-            final SSTableScanner scanner = sstable.getScanner();
+            final SSTableScanner scanner = sstable.getScanner(KEY_RANGE_FILE_BUFFER_SIZE);
             scanner.seekTo(startWith);
             Iterator<DecoratedKey> iter = new CloseableIterator<DecoratedKey>()
             {

Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
(original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/CompactionIterator.java
Wed Dec 23 17:45:58 2009
@@ -40,6 +40,8 @@
 {
     private static Logger logger = Logger.getLogger(CompactionIterator.class);
 
+    private static final int FILE_BUFFER_SIZE = 1024 * 1024;
+
     private final List<IteratingRow> rows = new ArrayList<IteratingRow>();
     private final int gcBefore;
     private boolean major;
@@ -65,7 +67,7 @@
         });
         for (SSTableReader sstable : sstables)
         {
-            iter.addIterator(sstable.getScanner());
+            iter.addIterator(sstable.getScanner(FILE_BUFFER_SIZE));
         }
         return iter;
     }

Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
(original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableReader.java
Wed Dec 23 17:45:58 2009
@@ -403,9 +403,9 @@
         return partitioner;
     }
 
-    public SSTableScanner getScanner() throws IOException
+    public SSTableScanner getScanner(int bufferSize) throws IOException
     {
-        return new SSTableScanner(this);
+        return new SSTableScanner(this, bufferSize);
     }
 
     public AbstractType getColumnComparator()

Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
(original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/io/SSTableScanner.java
Wed Dec 23 17:45:58 2009
@@ -38,13 +38,13 @@
     private SSTableReader sstable;
     private Iterator<IteratingRow> iterator;
 
-    SSTableScanner(SSTableReader sstable) throws IOException
+    /**
+     * @param sstable SSTable to scan.
+     * @param bufferSize Number of bytes to buffer the file while scanning.
+     */
+    SSTableScanner(SSTableReader sstable, int bufferSize) throws IOException
     {
-        // TODO this is used for both compactions and key ranges.  the buffer sizes we want
-        // to use for these ops are very different.  here we are leaning towards the key-range
-        // use case since that is more common.  What we really want is to split those
-        // two uses of this class up.
-        this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 256 * 1024);
+        this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", bufferSize);
         this.sstable = sstable;
     }
 

Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java?rev=893584&r1=893583&r2=893584&view=diff
==============================================================================
--- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java
(original)
+++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/tools/SSTableExport.java
Wed Dec 23 17:45:58 2009
@@ -40,6 +40,8 @@
  */
 public class SSTableExport
 {
+    private static int INPUT_FILE_BUFFER_SIZE = 8 * 1024 * 1024;
+
     private static final String OUTFILE_OPTION = "f";
     private static final String KEY_OPTION = "k";
     private static Options options;
@@ -143,7 +145,7 @@
     throws IOException
     {
         SSTableReader reader = SSTableReader.open(ssTableFile);
-        SSTableScanner scanner = reader.getScanner();
+        SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE);
         IPartitioner<?> partitioner = DatabaseDescriptor.getPartitioner();    
         int i = 0;
         
@@ -201,7 +203,7 @@
     // than once from within the same process.
     static void export(SSTableReader reader, PrintStream outs) throws IOException
     {
-        SSTableScanner scanner = reader.getScanner();
+        SSTableScanner scanner = reader.getScanner(INPUT_FILE_BUFFER_SIZE);
         
         outs.println("{");
         
@@ -321,4 +323,4 @@
         }
         System.exit(0);
     }
-}
\ No newline at end of file
+}



Mime
View raw message