incubator-cassandra-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Norman Maurer <norman.mau...@googlemail.com>
Subject Re: svn commit: r1180970 - in /cassandra/branches/cassandra-1.0.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Date Mon, 10 Oct 2011 14:03:34 GMT
Hi there,

did you alter the snappy-java version by mistake or was this part of
the commit ?

Bye,
Norman


2011/10/10  <slebresne@apache.org>:
> Author: slebresne
> Date: Mon Oct 10 13:56:27 2011
> New Revision: 1180970
>
> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev
> Log:
> Fix places where uncompressed sstable size is used in place of the compressed one.
> patch by slebresne; reviewed by jbellis for CASSANDRA-3338
>
> Modified:
>    cassandra/branches/cassandra-1.0.0/CHANGES.txt
>    cassandra/branches/cassandra-1.0.0/build.xml
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
>    cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
>
> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original)
> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011
> @@ -12,6 +12,8 @@
>  * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308)
>  * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329)
>  * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331)
> + * Fix places where uncompressed size of sstables was use in place of the
> +   compressed one (CASSANDRA-3338)
>  Fixes merged from 0.8 below:
>  * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258)
>  * Force flush of status table when removing/updating token (CASSANDRA-3243)
>
> Modified: cassandra/branches/cassandra-1.0.0/build.xml
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/build.xml (original)
> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011
> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry
>         <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt"/>
>         <scm connection="${scm.connection}" developerConnection="${scm.developerConnection}"
url="${scm.url}"/>
>         <dependencyManagement>
> -          <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3.3"/>
> +          <dependency groupId="org.xerial.snappy" artifactId="snappy-java" version="1.0.3"/>
>           <dependency groupId="com.ning" artifactId="compress-lzf" version="0.8.4"/>
>           <dependency groupId="com.google.guava" artifactId="guava" version="r08"/>
>           <dependency groupId="commons-cli" artifactId="commons-cli" version="1.1"/>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Oct 10 13:56:27 2011
> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement
>         long expectedFileSize = 0;
>         for (SSTableReader sstable : sstables)
>         {
> -            long size = sstable.length();
> +            long size = sstable.onDiskLength();
>             expectedFileSize = expectedFileSize + size;
>         }
>         return expectedFileSize;
> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement
>         SSTableReader maxFile = null;
>         for (SSTableReader sstable : sstables)
>         {
> -            if (sstable.length() > maxSize)
> +            if (sstable.onDiskLength() > maxSize)
>             {
> -                maxSize = sstable.length();
> +                maxSize = sstable.onDiskLength();
>                 maxFile = sstable;
>             }
>         }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Mon Oct 10 13:56:27 2011
> @@ -472,7 +472,7 @@ public class CompactionManager implement
>         boolean isCommutative = cfs.metadata.getDefaultValidator().isCommutative();
>
>         // Calculate the expected compacted filesize
> -        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.length());
> +        String compactionFileLocation = cfs.table.getDataFileLocation(sstable.onDiskLength());
>         if (compactionFileLocation == null)
>             throw new IOException("disk full");
>         int expectedBloomFilterSize = Math.max(DatabaseDescriptor.getIndexInterval(),
> @@ -765,8 +765,8 @@ public class CompactionManager implement
>
>                 String format = "Cleaned up to %s.  %,d to %,d (~%d%% of original)
bytes for %,d keys.  Time: %,dms.";
>                 long dTime = System.currentTimeMillis() - startTime;
> -                long startsize = sstable.length();
> -                long endsize = newSstable.length();
> +                long startsize = sstable.onDiskLength();
> +                long endsize = newSstable.onDiskLength();
>                 double ratio = (double)endsize / (double)startsize;
>                 logger.info(String.format(format, writer.getFilename(), startsize,
endsize, (int)(ratio*100), totalkeysWritten, dTime));
>             }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java
Mon Oct 10 13:56:27 2011
> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg
>     {
>         List<Pair<SSTableReader, Long>> tableLengthPairs = new ArrayList<Pair<SSTableReader,
Long>>();
>         for(SSTableReader table: collection)
> -            tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.length()));
> +            tableLengthPairs.add(new Pair<SSTableReader, Long>(table, table.onDiskLength()));
>         return tableLengthPairs;
>     }
>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Mon Oct 10 13:56:27 2011
> @@ -257,7 +257,7 @@ public abstract class SSTable
>         long sum = 0;
>         for (SSTableReader sstable : sstables)
>         {
> -            sum += sstable.length();
> +            sum += sstable.onDiskLength();
>         }
>         return sum;
>     }
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Mon Oct 10 13:56:27 2011
> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab
>             long right = getPosition(new DecoratedKey(range.right, null), Operator.GT);
>             if (right == -1 || Range.isWrapAround(range.left, range.right))
>                 // right is past the end of the file, or it wraps
> -                right = length();
> +                right = uncompressedLength();
>             if (left == right)
>                 // empty range
>                 continue;
> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab
>     }
>
>     /**
> -     * @return The length in bytes of the data file for this SSTable.
> +     * @return The length in bytes of the data for this SSTable. For
> +     * compressed files, this is not the same thing as the on disk size (see
> +     * onDiskLength())
>      */
> -    public long length()
> +    public long uncompressedLength()
>     {
>         return dfile.length;
>     }
>
> +    /**
> +     * @return The length in bytes of the on disk size for this SSTable. For
> +     * compressed files, this is not the same thing as the data length (see
> +     * length())
> +     */
> +    public long onDiskLength()
> +    {
> +        return dfile.onDiskLength;
> +    }
> +
>     public boolean acquireReference()
>     {
>         while (true)
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java
Mon Oct 10 13:56:27 2011
> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext
>
>     public CompressedSegmentedFile(String path, CompressionMetadata metadata)
>     {
> -        super(path, metadata.dataLength);
> +        super(path, metadata.dataLength, metadata.compressedFileLength);
>         this.metadata = metadata;
>     }
>
>
> Modified: cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
> URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff
> ==============================================================================
> --- cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
(original)
> +++ cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java
Mon Oct 10 13:56:27 2011
> @@ -42,13 +42,23 @@ public abstract class SegmentedFile
>     public final String path;
>     public final long length;
>
> +    // This differs from length for compressed files (but we still need length for
> +    // SegmentIterator because offsets in the file are relative to the uncompressed
size)
> +    public final long onDiskLength;
> +
>     /**
>      * Use getBuilder to get a Builder to construct a SegmentedFile.
>      */
>     SegmentedFile(String path, long length)
>     {
> +        this(path, length, length);
> +    }
> +
> +    protected SegmentedFile(String path, long length, long onDiskLength)
> +    {
>         this.path = path;
>         this.length = length;
> +        this.onDiskLength = onDiskLength;
>     }
>
>     /**
>
>
>

Mime
View raw message