incubator-cassandra-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Norman Maurer <norman.mau...@googlemail.com>
Subject Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Date Mon, 10 Oct 2011 14:19:30 GMT
No problem... It just catched my attention ;)

Keep on coding,
Norman

2011/10/10 Sylvain Lebresne <sylvain@datastax.com>:
> On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer
> <norman.maurer@googlemail.com> wrote:
>> Hi there,
>>
>> did you alter the snappy-java version by mistake or was this part of
>> the commit ?
>
> Sorry, that was intentional but I should probably not have been lazy
> and made a separate commit.
> I just updated the version referenced in the pom so that it matches
> what we are using. i.e, I
> haven't updated the lib, I've just made it so that user of maven
> central will get the 'right' version.
>
> --
> Sylvain
>
>>
>> Bye,
>> Norman
>>
>>
>> 2011/10/10  <slebresne@apache.org>:
>>> Author: slebresne
>>> Date: Mon Oct 10 13:56:27 2011
>>> New Revision: 1180970
>>>
>>> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
>>> Log:
>>> Fix places where uncompressed sstable size is used in place of the compressed
one.
>>> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>>>
>>> Modified:
>>>    cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>>    cassandra/branches/cassandra-1.0.0/build.xml
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
>>> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
>>> @@ -12,6 +12,8 @@
>>>  * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>>>  * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>>>  * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
>>> + * Fix places where uncompressed size of sstables was use in place of the
>>> +   compressed one (CASSANDRA-3338)
>>>  Fixes merged from 0.8 below:
>>>  * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>>>  * Force flush of status table when removing/updating token (CASSANDRA-3243)
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/build.xml
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
>>> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
>>> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>>>         <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>>>         <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}"
url="${scm.url}"/>
>>>         <dependencyManagement>
>>> -          <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3.3"/>
>>> +          <dependency groupId="org.xerial.snappy" artifactId="snappy-java"
version="1.0.3"/>
>>>           <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>>>           <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>>>           <dependency groupId="commons-cli" artifactId="commons-cli"
version="1.1"/>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Oct 10 13:56:27 2011
>>> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>>>         long expectedFileSize = 0;
>>>         for (SSTableReader sstable : sstables)
>>>         {
>>> -            long size = sstable.length();
>>> +            long size = sstable.onDiskLength();
>>>             expectedFileSize = expectedFileSize + size;
>>>         }
>>>         return expectedFileSize;
>>> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>>>         SSTableReader maxFile = null;
>>>         for (SSTableReader sstable : sstables)
>>>         {
>>> -            if (sstable.length() > maxSize)
>>> +            if (sstable.onDiskLength() > maxSize)
>>>             {
>>> -                maxSize = sstable.length();
>>> +                maxSize = sstable.onDiskLength();
>>>                 maxFile = sstable;
>>>             }
>>>         }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Mon Oct 10 13:56:27 2011
>>> @@ -472,7 +472,7 @@ public class CompactionManager implement
>>>         boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>>>
>>>         // Calculate the expected compacted filesize
>>> -        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
>>> +        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>>>         if (compactionFileLocation == null)
>>>             throw new IOException("disk full");
>>>         int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
>>> @@ -765,8 +765,8 @@ public class CompactionManager implement
>>>
>>>                 String format = "Cleaned up to %s.  %,d to %,d (~%d%%
of original) bytes for %,d keys.  Time: %,dms.";
>>>                 long dTime = System.currentTimeMillis() - startTime;
>>> -                long startsize = sstable.length();
>>> -                long endsize = newSstable.length();
>>> +                long startsize = sstable.onDiskLength();
>>> +                long endsize = newSstable.onDiskLength();
>>>                 double ratio = (double)endsize / (double)startsize;
>>>                 logger.info(String.format(format, writer.getFilename(),
startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime));
>>>             }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Mon Oct 10 13:56:27 2011
>>> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>>>     {
>>>         List<Pair<SSTableReader, Long>> tableLengthPairs = new
ArrayList<Pair<SSTableReader, Long>>();
>>>         for(SSTableReader table: collection)
>>> -            tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.length()));
>>> +            tableLengthPairs.add(new Pair<SSTableReader, Long>(table,
table.onDiskLength()));
>>>         return tableLengthPairs;
>>>     }
>>>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Mon Oct 10 13:56:27 2011
>>> @@ -257,7 +257,7 @@ public abstract class SSTable
>>>         long sum = 0;
>>>         for (SSTableReader sstable : sstables)
>>>         {
>>> -            sum += sstable.length();
>>> +            sum += sstable.onDiskLength();
>>>         }
>>>         return sum;
>>>     }
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Oct 10 13:56:27 2011
>>> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>>>             long right = getPosition(new DecoratedKey(range.right, null),
Operator.GT);
>>>             if (right == -1 || Range.isWrapAround(range.left, range.right))
>>>                 // right is past the end of the file, or it wraps
>>> -                right = length();
>>> +                right = uncompressedLength();
>>>             if (left == right)
>>>                 // empty range
>>>                 continue;
>>> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>>>     }
>>>
>>>     /**
>>> -     * @return The length in bytes of the data file for this SSTable.
>>> +     * @return The length in bytes of the data for this SSTable. For
>>> +     * compressed files, this is not the same thing as the on disk size (see
>>> +     * onDiskLength())
>>>      */
>>> -    public long length()
>>> +    public long uncompressedLength()
>>>     {
>>>         return dfile.length;
>>>     }
>>>
>>> +    /**
>>> +     * @return The length in bytes of the on disk size for this SSTable. For
>>> +     * compressed files, this is not the same thing as the data length (see
>>> +     * length())
>>> +     */
>>> +    public long onDiskLength()
>>> +    {
>>> +        return dfile.onDiskLength;
>>> +    }
>>> +
>>>     public boolean acquireReference()
>>>     {
>>>         while (true)
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
Mon Oct 10 13:56:27 2011
>>> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>>>
>>>     public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>>>     {
>>> -        super(path, metadata.dataLength);
>>> +        super(path, metadata.dataLength, metadata.compressedFileLength);
>>>         this.metadata = metadata;
>>>     }
>>>
>>>
>>> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>>> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
>>> ==============================================================================
>>> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
(original)
>>> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Mon Oct 10 13:56:27 2011
>>> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>>>     public final String path;
>>>     public final long length;
>>>
>>> +    // This differs from length for compressed files (but we still need length
for
>>> +    // SegmentIterator because offsets in the file are relative to the uncompressed
size)
>>> +    public final long onDiskLength;
>>> +
>>>     /**
>>>      * Use getBuilder to get a Builder to construct a SegmentedFile.
>>>      */
>>>     SegmentedFile(String path, long length)
>>>     {
>>> +        this(path, length, length);
>>> +    }
>>> +
>>> +    protected SegmentedFile(String path, long length, long onDiskLength)
>>> +    {
>>>         this.path = path;
>>>         this.length = length;
>>> +        this.onDiskLength = onDiskLength;
>>>     }
>>>
>>>     /**
>>>
>>>
>>>
>>
>

Mime
View raw message