cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject git commit: Fix LCS splitting sstables based on uncompressed length
Date Thu, 12 Jul 2012 09:06:16 GMT
Updated Branches:
  refs/heads/cassandra-1.0 51a9fd131 -> 787e0e648


Fix LCS splitting sstables based on uncompressed length

patch by slebresne; reviewed by jbellis for CASSANDRA-4419


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/787e0e64
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/787e0e64
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/787e0e64

Branch: refs/heads/cassandra-1.0
Commit: 787e0e648854634d5457dde6792acfb3d97c3b66
Parents: 51a9fd1
Author: Sylvain Lebresne <sylvain@datastax.com>
Authored: Thu Jul 12 11:05:11 2012 +0200
Committer: Sylvain Lebresne <sylvain@datastax.com>
Committed: Thu Jul 12 11:05:11 2012 +0200

----------------------------------------------------------------------
 CHANGES.txt                                        |    1 +
 .../cassandra/db/compaction/CompactionTask.java    |    4 ++--
 .../db/compaction/LeveledCompactionTask.java       |    4 ++--
 .../io/compress/CompressedSequentialWriter.java    |    6 ++++++
 .../apache/cassandra/io/sstable/SSTableWriter.java |    5 +++++
 .../apache/cassandra/io/util/SequentialWriter.java |   12 ++++++++++++
 6 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d3cca6c..66a7b9e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@
  * Oversize integer in CQL throws NumberFormatException (CASSANDRA-4291)
  * Set gc_grace on index CF to 0 (CASSANDRA-4314)
  * fix 1.0.x node join to mixed version cluster, other nodes >= 1.1 (CASSANDRA-4195)
+ * Fix LCS splitting sstable base on uncompressed size (CASSANDRA-4419)
 
 
 1.0.10

http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
index 2a1b415..f554c7b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
@@ -169,7 +169,7 @@ public class CompactionTask extends AbstractCompactionTask
                         }
                     }
                 }
-                if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer, position))
+                if (!nni.hasNext() || newSSTableSegmentThresholdReached(writer))
                 {
                     SSTableReader toIndex = writer.closeAndOpenReader(getMaxDataAge(toCompact));
                     cachedKeyMap.put(toIndex, cachedKeys);
@@ -229,7 +229,7 @@ public class CompactionTask extends AbstractCompactionTask
     }
 
     //extensibility point for other strategies that may want to limit the upper bounds of
the sstable segment size
-    protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer, long position)
+    protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer) throws IOException
     {
         return false;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
index b0b605a..d872970 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionTask.java
@@ -69,9 +69,9 @@ public class LeveledCompactionTask extends CompactionTask
     }
 
     @Override
-    protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer, long position)
+    protected boolean newSSTableSegmentThresholdReached(SSTableWriter writer) throws IOException
     {
-        return position > sstableSizeInMB * 1024L * 1024L;
+        return writer.getOnDiskFilePointer() > sstableSizeInMB * 1024L * 1024L;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 545b09e..702a343 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -69,6 +69,12 @@ public class CompressedSequentialWriter extends SequentialWriter
     }
 
     @Override
+    public long getOnDiskFilePointer() throws IOException
+    {
+        return out.getFilePointer();
+    }
+
+    @Override
     public void sync() throws IOException
     {
         throw new UnsupportedOperationException();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index 53980ed..f08bb49 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -382,6 +382,11 @@ public class SSTableWriter extends SSTable
         return dataFile.getFilePointer();
     }
 
+    public long getOnDiskFilePointer() throws IOException
+    {
+        return dataFile.getOnDiskFilePointer();
+    }
+
     /**
      * Encapsulates writing the index and filter for an SSTable. The state of this object
is not valid until it has been closed.
      */

http://git-wip-us.apache.org/repos/asf/cassandra/blob/787e0e64/src/java/org/apache/cassandra/io/util/SequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index 79af8c1..dce9bc6 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -211,6 +211,18 @@ public class SequentialWriter extends OutputStream
         return current;
     }
 
+    /**
+     * Return the current file pointer of the underlying on-disk file.
+     * Note that since write works by buffering data, the value of this will increase by
buffer
+     * size and not every write to the writer will modify this value.
+     * Furthermore, for compressed files, this value refers to compressed data, while the
+     * writer getFilePointer() refers to uncompressedFile
+     */
+    public long getOnDiskFilePointer() throws IOException
+    {
+        return getFilePointer();
+    }
+
     public long length() throws IOException
     {
         return Math.max(Math.max(current, out.length()), bufferOffset + validBufferBytes);


Mime
View raw message