cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Date Mon, 10 Oct 2011 13:56:28 GMT
Author: slebresne
Date: Mon Oct 10 13:56:27 2011
New Revision: 1180970

URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
Log:
Fix places where uncompressed sstable size is used in place of the compressed one.
patch by slebresne; reviewed by jbellis for CASSANDRA-3338

Modified:
    cassandra/branches/cassandra-1.0.0/CHANGES.txt
    cassandra/branches/cassandra-1.0.0/build.xml
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java

Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
@@ -12,6 +12,8 @@
  * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
  * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
  * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
+ * Fix places where uncompressed size of sstables was use in place of the
+   compressed one (CASSANDRA-3338)
 Fixes merged from 0.8 below:
  * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
  * Force flush of status table when removing/updating token (CASSANDRA-3243)

Modified: cassandra/branches/cassandra-1.0.0/build.xml
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/build.xml (original)
+++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
@@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
         <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
         <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}"
url="${scm.url}"/>
         <dependencyManagement>
-          <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
+          <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
           <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
           <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
           <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Oct 10 13:56:27 2011
@@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
         long expectedFileSize = 0;
         for (SSTableReader sstable : sstables)
         {
-            long size = sstable.length();
+            long size = sstable.onDiskLength();
             expectedFileSize = expectedFileSize + size;
         }
         return expectedFileSize;
@@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
         SSTableReader maxFile = null;
         for (SSTableReader sstable : sstables)
         {
-            if (sstable.length() > maxSize)
+            if (sstable.onDiskLength() > maxSize)
             {
-                maxSize = sstable.length();
+                maxSize = sstable.onDiskLength();
                 maxFile = sstable;
             }
         }

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Mon Oct 10 13:56:27 2011
@@ -472,7 +472,7 @@ public class CompactionManager implement
         boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
 
         // Calculate the expected compacted filesize
-        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
+        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
         if (compactionFileLocation == null)
             throw new IOException("disk full");
         int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
@@ -765,8 +765,8 @@ public class CompactionManager implement
 
                 String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original) bytes
for %,d keys.  Time: %,dms.";
                 long dTime = System.currentTimeMillis() - startTime;
-                long startsize = sstable.length();
-                long endsize = newSstable.length();
+                long startsize = sstable.onDiskLength();
+                long endsize = newSstable.onDiskLength();
                 double ratio = (double)endsize / (double)startsize;
                 logger.info(String.format(format, writer.getFilename(), startsize, endsize,
(int)(ratio*100), totalkeysWritten, dTime));
             }

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Mon Oct 10 13:56:27 2011
@@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
     {
         List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader,
Long>>();
         for(SSTableReader table: collection)
-            tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
+            tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
         return tableLengthPairs;
     }
 

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Mon Oct 10 13:56:27 2011
@@ -257,7 +257,7 @@ public abstract class SSTable
         long sum = 0;
         for (SSTableReader sstable : sstables)
         {
-            sum += sstable.length();
+            sum += sstable.onDiskLength();
         }
         return sum;
     }

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Oct 10 13:56:27 2011
@@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
             long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
             if (right == -1 || Range.isWrapAround(range.left, range.right))
                 // right is past the end of the file, or it wraps
-                right = length();
+                right = uncompressedLength();
             if (left == right)
                 // empty range
                 continue;
@@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
     }
 
     /**
-     * @return The length in bytes of the data file for this SSTable.
+     * @return The length in bytes of the data for this SSTable. For
+     * compressed files, this is not the same thing as the on disk size (see
+     * onDiskLength())
      */
-    public long length()
+    public long uncompressedLength()
     {
         return dfile.length;
     }
 
+    /**
+     * @return The length in bytes of the on disk size for this SSTable. For
+     * compressed files, this is not the same thing as the data length (see
+     * length())
+     */
+    public long onDiskLength()
+    {
+        return dfile.onDiskLength;
+    }
+
     public boolean acquireReference()
     {
         while (true)

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
Mon Oct 10 13:56:27 2011
@@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
 
     public CompressedSegmentedFile(String path, CompressionMetadata metadata)
     {
-        super(path, metadata.dataLength);
+        super(path, metadata.dataLength, metadata.compressedFileLength);
         this.metadata = metadata;
     }
 

Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
(original)
+++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Mon Oct 10 13:56:27 2011
@@ -42,13 +42,23 @@ public abstract class SegmentedFile
     public final String path;
     public final long length;
 
+    // This differs from length for compressed files (but we still need length for
+    // SegmentIterator because offsets in the file are relative to the uncompressed size)
+    public final long onDiskLength;
+
     /**
      * Use getBuilder to get a Builder to construct a SegmentedFile.
      */
     SegmentedFile(String path, long length)
     {
+        this(path, length, length);
+    }
+
+    protected SegmentedFile(String path, long length, long onDiskLength)
+    {
         this.path = path;
         this.length = length;
+        this.onDiskLength = onDiskLength;
     }
 
     /**



Mime
View raw message