incubator-cassandra-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sylvain Lebresne <sylv...@datastax.com>
Subject Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Date Mon, 10 Oct 2011 14:17:18 GMT
On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer
<norman.maurer@googlemail.com> wrote:
> Hi there,
>
> did you alter the snappy-java version by mistake or was this part of
> the commit ?

Sorry, that was intentional but I should probably not have been lazy
and made a separate commit.
I just updated the version referenced in the pom so that it matches
what we are using. i.e, I
haven't updated the lib, I've just made it so that user of maven
central will get the 'right' version.

--
Sylvain

>
> Bye,
> Norman
>
>
> 2011/10/10  <slebresne@apache.org>:
>> Author: slebresne
>> Date: Mon Oct 10 13:56:27 2011
>> New Revision: 1180970
>>
>> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
>> Log:
>> Fix places where uncompressed sstable size is used in place of the compressed one.
>> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>>
>> Modified:
>>    cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>    cassandra/branches/cassandra-1.0.0/build.xml
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>
>> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
>> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
>> @@ -12,6 +12,8 @@
>>  * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>>  * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>>  * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
>> + * Fix places where uncompressed size of sstables was use in place of the
>> +   compressed one (CASSANDRA-3338)
>>  Fixes merged from 0.8 below:
>>  * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>>  * Force flush of status table when removing/updating token (CASSANDRA-3243)
>>
>> Modified: cassandra/branches/cassandra-1.0.0/build.xml
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
>> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
>> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>>         <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>>         <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}"
url="${scm.url}"/>
>>         <dependencyManagement>
>> -          <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3.3"/>
>> +          <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3"/>
>>           <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>>           <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>>           <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Oct 10 13:56:27 2011
>> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>>         long expectedFileSize = 0;
>>         for (SSTableReader sstable : sstables)
>>         {
>> -            long size = sstable.length();
>> +            long size = sstable.onDiskLength();
>>             expectedFileSize = expectedFileSize + size;
>>         }
>>         return expectedFileSize;
>> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>>         SSTableReader maxFile = null;
>>         for (SSTableReader sstable : sstables)
>>         {
>> -            if (sstable.length() > maxSize)
>> +            if (sstable.onDiskLength() > maxSize)
>>             {
>> -                maxSize = sstable.length();
>> +                maxSize = sstable.onDiskLength();
>>                 maxFile = sstable;
>>             }
>>         }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Mon Oct 10 13:56:27 2011
>> @@ -472,7 +472,7 @@ public class CompactionManager implement
>>         boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>>
>>         // Calculate the expected compacted filesize
>> -        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
>> +        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>>         if (compactionFileLocation == null)
>>             throw new IOException("disk full");
>>         int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
>> @@ -765,8 +765,8 @@ public class CompactionManager implement
>>
>>                 String format = "Cleaned up to %s.  %,d to %,d (~%d%% of
original) bytes for %,d keys.  Time: %,dms.";
>>                 long dTime = System.currentTimeMillis() - startTime;
>> -                long startsize = sstable.length();
>> -                long endsize = newSstable.length();
>> +                long startsize = sstable.onDiskLength();
>> +                long endsize = newSstable.onDiskLength();
>>                 double ratio = (double)endsize / (double)startsize;
>>                 logger.info(String.format(format, writer.getFilename(), startsize,
endsize, (int)(ratio*100), totalkeysWritten, dTime));
>>             }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Mon Oct 10 13:56:27 2011
>> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>>     {
>>         List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader,
Long>>();
>>         for(SSTableReader table: collection)
>> -            tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.length()));
>> +            tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.onDiskLength()));
>>         return tableLengthPairs;
>>     }
>>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Mon Oct 10 13:56:27 2011
>> @@ -257,7 +257,7 @@ public abstract class SSTable
>>         long sum = 0;
>>         for (SSTableReader sstable : sstables)
>>         {
>> -            sum += sstable.length();
>> +            sum += sstable.onDiskLength();
>>         }
>>         return sum;
>>     }
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Oct 10 13:56:27 2011
>> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>>             long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
>>             if (right == -1 || Range.isWrapAround(range.left, range.right))
>>                 // right is past the end of the file, or it wraps
>> -                right = length();
>> +                right = uncompressedLength();
>>             if (left == right)
>>                 // empty range
>>                 continue;
>> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>>     }
>>
>>     /**
>> -     * @return The length in bytes of the data file for this SSTable.
>> +     * @return The length in bytes of the data for this SSTable. For
>> +     * compressed files, this is not the same thing as the on disk size (see
>> +     * onDiskLength())
>>      */
>> -    public long length()
>> +    public long uncompressedLength()
>>     {
>>         return dfile.length;
>>     }
>>
>> +    /**
>> +     * @return The length in bytes of the on disk size for this SSTable. For
>> +     * compressed files, this is not the same thing as the data length (see
>> +     * length())
>> +     */
>> +    public long onDiskLength()
>> +    {
>> +        return dfile.onDiskLength;
>> +    }
>> +
>>     public boolean acquireReference()
>>     {
>>         while (true)
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
Mon Oct 10 13:56:27 2011
>> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>>
>>     public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>>     {
>> -        super(path, metadata.dataLength);
>> +        super(path, metadata.dataLength, metadata.compressedFileLength);
>>         this.metadata = metadata;
>>     }
>>
>>
>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>> ==============================================================================
>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
(original)
>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Mon Oct 10 13:56:27 2011
>> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>>     public final String path;
>>     public final long length;
>>
>> +    // This differs from length for compressed files (but we still need length
for
>> +    // SegmentIterator because offsets in the file are relative to the uncompressed
size)
>> +    public final long onDiskLength;
>> +
>>     /**
>>      * Use getBuilder to get a Builder to construct a SegmentedFile.
>>      */
>>     SegmentedFile(String path, long length)
>>     {
>> +        this(path, length, length);
>> +    }
>> +
>> +    protected SegmentedFile(String path, long length, long onDiskLength)
>> +    {
>>         this.path = path;
>>         this.length = length;
>> +        this.onDiskLength = onDiskLength;
>>     }
>>
>>     /**
>>
>>
>>
>

Mime
View raw message