cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xe...@apache.org
Subject svn commit: r1199427 - in /cassandra/branches/cassandra-1.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/compress/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/tools/ test/unit/org/apache/cassandra/i...
Date Tue, 08 Nov 2011 20:04:18 GMT
Author: xedin
Date: Tue Nov  8 20:04:17 2011
New Revision: 1199427

URL: http://svn.apache.org/viewvc?rev=1199427&view=rev
Log:
Report compression ratio in CFSMBean
patch by vijay; reviewed by xedin for CASSANDRA-3393

Modified:
    cassandra/branches/cassandra-1.0/CHANGES.txt
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
    cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java

Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Tue Nov  8 20:04:17 2011
@@ -13,7 +13,7 @@ Merged from 0.8:
  * Make counter shard merging thread safe (CASSANDRA-3178)
  * Revert CASSANDRA-2855
  * Fix bug preventing the use of efficient cross-DC writes (CASSANDRA-3472)
-
+ * Report compression ratio in CFSMBean (CASSANDRA-3393)
 
 1.0.2
  * "defragment" rows for name-based queries under STCS (CASSANDRA-2503)

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Tue Nov  8 20:04:17 2011
@@ -1784,6 +1784,12 @@ public class ColumnFamilyStore implement
         return data.getEstimatedColumnCountHistogram();
     }
 
+    @Override
+    public double getCompressionRatio()
+    {
+        return data.getCompressionRatio();
+    }
+
     /** true if this CFS contains secondary index data */
     public boolean isIndex()
     {

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
Tue Nov  8 20:04:17 2011
@@ -213,6 +213,7 @@ public interface ColumnFamilyStoreMBean
 
     public long[] getEstimatedRowSizeHistogram();
     public long[] getEstimatedColumnCountHistogram();
+    public double getCompressionRatio();
 
     /**
      * Returns a list of the names of the built column indexes for current store

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java Tue
Nov  8 20:04:17 2011
@@ -364,6 +364,21 @@ public class DataTracker
         return histogram;
     }
 
+    public double getCompressionRatio()
+    {
+        double sum = 0;
+        int total = 0;
+        for (SSTableReader sstable : getSSTables())
+        {
+            if (sstable.getCompressionRatio() != Double.MIN_VALUE)
+            {
+                sum += sstable.getCompressionRatio();
+                total++;
+            }
+        }
+        return total != 0 ? (double)sum/total: 0;
+    }
+
     public long getMinRowSize()
     {
         long min = 0;

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
Tue Nov  8 20:04:17 2011
@@ -23,14 +23,15 @@ import java.io.IOException;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
+import org.apache.cassandra.io.sstable.SSTableMetadata.Collector;
 import org.apache.cassandra.io.util.FileMark;
 import org.apache.cassandra.io.util.SequentialWriter;
 
 public class CompressedSequentialWriter extends SequentialWriter
 {
-    public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters) throws IOException
+    public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters, Collector sstableMetadataCollector) throws
IOException
     {
-        return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache,
parameters);
+        return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache,
parameters, sstableMetadataCollector);
     }
 
     // holds offset in the file where current chunk should be written
@@ -49,7 +50,11 @@ public class CompressedSequentialWriter 
 
     private final Checksum checksum = new CRC32();
 
-    public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache,
CompressionParameters parameters) throws IOException
+    private long originalSize = 0, compressedSize = 0;
+
+    private Collector sstableMetadataCollector;
+
+    public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache,
CompressionParameters parameters, Collector sstableMetadataCollector) throws IOException
     {
         super(file, parameters.chunkLength(), skipIOCache);
         this.compressor = parameters.sstableCompressor;
@@ -60,6 +65,7 @@ public class CompressedSequentialWriter 
         /* Index File (-CompressionInfo.db component) and it's header */
         metadataWriter = new CompressionMetadata.Writer(indexFilePath);
         metadataWriter.writeHeader(parameters);
+        this.sstableMetadataCollector = sstableMetadataCollector;
     }
 
     @Override
@@ -82,6 +88,9 @@ public class CompressedSequentialWriter 
         // compressing data with buffer re-use
         int compressedLength = compressor.compress(buffer, 0, validBufferBytes, compressed,
0);
 
+        originalSize += validBufferBytes;
+        compressedSize += compressedLength;
+
         // update checksum
         checksum.update(buffer, 0, validBufferBytes);
 
@@ -179,7 +188,7 @@ public class CompressedSequentialWriter 
             return; // already closed
 
         super.close();
-
+        sstableMetadataCollector.addCompressionRatio(compressedSize, originalSize);
         metadataWriter.finalizeHeader(current, chunkCount);
         metadataWriter.close();
     }

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
Tue Nov  8 20:04:17 2011
@@ -48,7 +48,7 @@ public class Descriptor
     // f (0.7.0): switched bloom filter implementations in data component
     // g (0.8): tracks flushed-at context in metadata component
     // h (1.0): tracks max client timestamp in metadata component
-    public static final String CURRENT_VERSION = "h";
+    public static final String CURRENT_VERSION = "i";
 
     public final File directory;
     public final String version;
@@ -112,6 +112,11 @@ public class Descriptor
         return version.compareTo("g") >= 0;
     }
 
+    public boolean hasCompressionRatio()
+    {
+        return version.compareTo("i") >= 0;
+    }
+
     public String filenameFor(Component component)
     {
         return filenameFor(component.name());

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
Tue Nov  8 20:04:17 2011
@@ -53,6 +53,7 @@ public class SSTableMetadata
     protected final EstimatedHistogram estimatedColumnCount;
     protected final ReplayPosition replayPosition;
     protected final long maxTimestamp;
+    protected final double compressionRatio;
     public static final SSTableMetadataSerializer serializer = new SSTableMetadataSerializer();
 
     private SSTableMetadata()
@@ -63,15 +64,16 @@ public class SSTableMetadata
     // when there is no max timestamp recorded, default to max long
     private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition)
     {
-        this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE);
+        this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE, Double.MIN_VALUE);
     }
 
-    private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition, long maxTimestamp)
+    private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition, long maxTimestamp, double cr)
     {
         this.estimatedRowSize = rowSizes;
         this.estimatedColumnCount = columnCounts;
         this.replayPosition = replayPosition;
         this.maxTimestamp = maxTimestamp;
+        this.compressionRatio = cr;
     }
 
     public static SSTableMetadata createDefaultInstance()
@@ -104,6 +106,11 @@ public class SSTableMetadata
         return maxTimestamp;
     }
 
+    public double getCompressionRatio()
+    {
+        return compressionRatio;
+    }
+
     static EstimatedHistogram defaultColumnCountHistogram()
     {
         // EH of 114 can track a max value of 2395318855, i.e., > 2B columns
@@ -122,6 +129,7 @@ public class SSTableMetadata
         protected EstimatedHistogram estimatedColumnCount;
         protected ReplayPosition replayPosition;
         protected long maxTimestamp;
+        protected double compressionRatio;
 
         private Collector()
         {
@@ -129,6 +137,7 @@ public class SSTableMetadata
             this.estimatedColumnCount = defaultColumnCountHistogram();
             this.replayPosition = ReplayPosition.NONE;
             this.maxTimestamp = Long.MIN_VALUE;
+            this.compressionRatio = Double.MIN_VALUE;
         }
 
         public void addRowSize(long rowSize)
@@ -141,6 +150,15 @@ public class SSTableMetadata
             estimatedColumnCount.add(columnCount);
         }
 
+        /**
+         * Ratio is compressed/uncompressed and it is
+         * if you have 1.x then compression isn't helping
+         */
+        public void addCompressionRatio(long compressed, long uncompressed)
+        {
+            compressionRatio = (double) compressed/uncompressed;
+        }
+
         public void updateMaxTimestamp(long potentialMax)
         {
             maxTimestamp = Math.max(maxTimestamp, potentialMax);
@@ -148,7 +166,7 @@ public class SSTableMetadata
 
         public SSTableMetadata finalizeMetadata()
         {
-            return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition,
maxTimestamp);
+            return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition,
maxTimestamp, compressionRatio);
         }
 
         public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@@ -180,6 +198,7 @@ public class SSTableMetadata
             EstimatedHistogram.serializer.serialize(sstableStats.getEstimatedColumnCount(),
dos);
             ReplayPosition.serializer.serialize(sstableStats.getReplayPosition(), dos);
             dos.writeLong(sstableStats.getMaxTimestamp());
+            dos.writeDouble(sstableStats.getCompressionRatio());
         }
 
         public SSTableMetadata deserialize(Descriptor descriptor) throws IOException
@@ -205,8 +224,14 @@ public class SSTableMetadata
                 ReplayPosition replayPosition = descriptor.hasReplayPosition()
                                               ? ReplayPosition.serializer.deserialize(dis)
                                               : ReplayPosition.NONE;
-
-                return new SSTableMetadata(rowSizes, columnCounts, replayPosition);
+                long maxTimestamp = Long.MAX_VALUE;
+                double compressionRatio = Double.MIN_VALUE;
+                if (descriptor.hasCompressionRatio())
+                {
+                    maxTimestamp = dis.readLong();
+                    compressionRatio = dis.readDouble();
+                }
+                return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp,
compressionRatio);
             }
             finally
             {
@@ -220,7 +245,7 @@ public class SSTableMetadata
             EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis);
             ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(dis);
             long maxTimestamp = dis.readLong();
-            return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp);
+            return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp,
Double.MIN_VALUE);
         }
 
         public long serializedSize(SSTableMetadata object)

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Tue Nov  8 20:04:17 2011
@@ -894,6 +894,11 @@ public class SSTableReader extends SSTab
         return sstableMetadata.getEstimatedColumnCount();
     }
 
+    public double getCompressionRatio()
+    {
+        return sstableMetadata.getCompressionRatio();
+    }
+
     public ReplayPosition getReplayPosition()
     {
         return sstableMetadata.getReplayPosition();

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Tue Nov  8 20:04:17 2011
@@ -87,7 +87,8 @@ public class SSTableWriter extends SSTab
             dataFile = CompressedSequentialWriter.open(getFilename(),
                                                        descriptor.filenameFor(Component.COMPRESSION_INFO),
                                                        true,
-                                                       metadata.compressionParameters());
+                                                       metadata.compressionParameters(),
+                                                       sstableMetadataCollector);
         }
         else
         {

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java Tue
Nov  8 20:04:17 2011
@@ -489,6 +489,8 @@ public class NodeCmd
                 outs.println("\t\tCompacted row minimum size: " + cfstore.getMinRowSize());
                 outs.println("\t\tCompacted row maximum size: " + cfstore.getMaxRowSize());
                 outs.println("\t\tCompacted row mean size: " + cfstore.getMeanRowSize());
+                if (cfstore.getCompressionRatio() != 0)
+                    outs.println("\t\tSSTable Compression Ratio: " + cfstore.getCompressionRatio());
 
                 outs.println("");
             }

Modified: cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
(original)
+++ cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
Tue Nov  8 20:04:17 2011
@@ -24,6 +24,7 @@ import java.util.concurrent.Callable;
 
 import org.junit.Test;
 
+import org.apache.cassandra.io.sstable.SSTableMetadata;
 import org.apache.cassandra.io.util.*;
 
 import static org.junit.Assert.assertEquals;
@@ -54,8 +55,9 @@ public class CompressedRandomAccessReade
 
         try
         {
+            SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
             SequentialWriter writer = compressed
-                ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance))
+                ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance),
sstableMetadataCollector)
                 : new SequentialWriter(f, CompressionParameters.DEFAULT_CHUNK_LENGTH, false);
 
             writer.write("The quick ".getBytes());
@@ -103,8 +105,8 @@ public class CompressedRandomAccessReade
 
         File metadata = new File(file.getPath() + ".meta");
         metadata.deleteOnExit();
-
-        SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(),
false, new CompressionParameters(SnappyCompressor.instance));
+        SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
+        SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(),
false, new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector);
 
         writer.write(CONTENT.getBytes());
         writer.close();



Mime
View raw message