cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject cassandra git commit: Estimate compressed on disk size when deciding if sstable size limit reached
Date Wed, 11 May 2016 08:46:39 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 2d76bd4c6 -> 0f1e838d1


Estimate compressed on disk size when deciding if sstable size limit reached

Patch by Tom Petracca; reviewed by marcuse for CASSANDRA-11623


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0f1e838d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0f1e838d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0f1e838d

Branch: refs/heads/trunk
Commit: 0f1e838d17ac90d4d44e3369309b245afa969f33
Parents: 2d76bd4
Author: Tom Petracca <tpetracca@palantir.com>
Authored: Mon Apr 18 16:37:00 2016 -0400
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Wed May 11 10:43:44 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                              |  1 +
 .../compaction/writers/MajorLeveledCompactionWriter.java |  7 +++----
 .../db/compaction/writers/MaxSSTableSizeWriter.java      |  2 +-
 .../writers/SplittingSizeTieredCompactionWriter.java     |  3 +--
 .../io/compress/CompressedSequentialWriter.java          | 11 +++++++++++
 .../cassandra/io/sstable/format/SSTableWriter.java       |  5 +++++
 .../cassandra/io/sstable/format/big/BigTableWriter.java  |  5 +++++
 .../org/apache/cassandra/io/util/SequentialWriter.java   |  5 +++++
 8 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0b70d77..a9421b6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.8
+ * Estimate compressed on disk size when deciding if sstable size limit reached (CASSANDRA-11623)
  * cassandra-stress profiles should support case sensitive schemas (CASSANDRA-11546)
  * Remove DatabaseDescriptor dependency from FileUtils (CASSANDRA-11578)
  * Faster streaming (CASSANDRA-9766)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index 91a18da..6cccfcb 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -17,7 +17,6 @@
  */
 package org.apache.cassandra.db.compaction.writers;
 
-import java.util.List;
 import java.util.Set;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -82,12 +81,12 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     @SuppressWarnings("resource")
     public boolean realAppend(UnfilteredRowIterator partition)
     {
-        long posBefore = sstableWriter.currentWriter().getOnDiskFilePointer();
         RowIndexEntry rie = sstableWriter.append(partition);
-        totalWrittenInLevel += sstableWriter.currentWriter().getOnDiskFilePointer() - posBefore;
         partitionsWritten++;
-        if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
+        long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten();
+        if (totalWrittenInCurrentWriter > maxSSTableSize)
         {
+            totalWrittenInLevel += totalWrittenInCurrentWriter;
             if (totalWrittenInLevel > LeveledManifest.maxBytesForLevel(currentLevel, maxSSTableSize))
             {
                 totalWrittenInLevel = 0;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 8c3ada7..864185e 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -96,7 +96,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
     protected boolean realAppend(UnfilteredRowIterator partition)
     {
         RowIndexEntry rie = sstableWriter.append(partition);
-        if (sstableWriter.currentWriter().getOnDiskFilePointer() > maxSSTableSize)
+        if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize)
         {
             switchCompactionLocation(sstableDirectory);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 290e88b..46cb891 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction.writers;
 
 import java.util.Arrays;
-import java.util.List;
 import java.util.Set;
 
 import org.slf4j.Logger;
@@ -89,7 +88,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
     public boolean realAppend(UnfilteredRowIterator partition)
     {
         RowIndexEntry rie = sstableWriter.append(partition);
-        if (sstableWriter.currentWriter().getOnDiskFilePointer() > currentBytesToWrite
&& currentRatioIndex < ratios.length - 1) // if we underestimate how many keys
we have, the last sstable might get more than we expect
+        if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite
&& currentRatioIndex < ratios.length - 1) // if we underestimate how many keys
we have, the last sstable might get more than we expect
         {
             currentRatioIndex++;
             currentBytesToWrite = Math.round(totalSize * ratios[currentRatioIndex]);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
index 9bd1145..1f33d53 100644
--- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
@@ -92,6 +92,17 @@ public class CompressedSequentialWriter extends SequentialWriter
         }
     }
 
+    /**
+     * Get a quick estimation on how many bytes have been written to disk
+     *
+     * It should for the most part be exactly the same as getOnDiskFilePointer()
+     */
+    @Override
+    public long getEstimatedOnDiskBytesWritten()
+    {
+        return chunkOffset;
+    }
+
     @Override
     public void flush()
     {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index 6aaf776..9f2e159 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -212,6 +212,11 @@ public abstract class SSTableWriter extends SSTable implements Transactional
 
     public abstract long getOnDiskFilePointer();
 
+    public long getEstimatedOnDiskBytesWritten()
+    {
+        return getOnDiskFilePointer();
+    }
+
     public abstract void resetAndTruncate();
 
     public SSTableWriter setRepairedAt(long repairedAt)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 39dc889..8645158 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -394,6 +394,11 @@ public class BigTableWriter extends SSTableWriter
         return dataFile.getOnDiskFilePointer();
     }
 
+    public long getEstimatedOnDiskBytesWritten()
+    {
+        return dataFile.getEstimatedOnDiskBytesWritten();
+    }
+
     /**
      * Encapsulates writing the index and filter for an SSTable. The state of this object
is not valid until it has been closed.
      */

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0f1e838d/src/java/org/apache/cassandra/io/util/SequentialWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/SequentialWriter.java b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
index f153b4d..45e4cfa 100644
--- a/src/java/org/apache/cassandra/io/util/SequentialWriter.java
+++ b/src/java/org/apache/cassandra/io/util/SequentialWriter.java
@@ -276,6 +276,11 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements
Tr
         return position();
     }
 
+    public long getEstimatedOnDiskBytesWritten()
+    {
+        return getOnDiskFilePointer();
+    }
+
     public long length()
     {
         try


Mime
View raw message