cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ble...@apache.org
Subject cassandra git commit: Make compression ratio much more accurate
Date Thu, 26 Nov 2015 14:10:18 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 811ffa7bf -> fa92ae66d


Make compression ratio much more accurate

patch by Brett Snyder; reviewed by Benjamin Lerer for CASSANDRA-10225


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/fa92ae66
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/fa92ae66
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/fa92ae66

Branch: refs/heads/trunk
Commit: fa92ae66d3ac6966bd08e7eda06183695dc34905
Parents: 811ffa7
Author: Brett Snyder <bsnyder788@gmail.com>
Authored: Thu Nov 26 15:08:54 2015 +0100
Committer: Benjamin Lerer <b.lerer@gmail.com>
Committed: Thu Nov 26 15:08:54 2015 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 NEWS.txt                                        |  7 ++-
 .../apache/cassandra/metrics/TableMetrics.java  | 58 +++++++++++---------
 3 files changed, 37 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/fa92ae66/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a952d10..37e60cb 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.2
+ * Make compression ratio much more accurate (CASSANDRA-10225)
  * Optimize building of Clustering object when only one is created (CASSANDRA-10409)
  * Make index building pluggable (CASSANDRA-10681)
  * Add sstable flush observer (CASSANDRA-10678)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fa92ae66/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index 41cccb4..2ccc27d 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -20,8 +20,11 @@ New features
 ------------
 
    - Support for type casting has been added to the selection clause.
-   
-   
+
+Upgrading
+---------
+   - The compression ratio metrics computation has been modified to be more accurate.
+
 3.0
 ===
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/fa92ae66/src/java/org/apache/cassandra/metrics/TableMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java
index 3cd5b5b..fd69ea9 100644
--- a/src/java/org/apache/cassandra/metrics/TableMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java
@@ -24,13 +24,14 @@ import java.util.concurrent.TimeUnit;
 
 import com.codahale.metrics.*;
 import com.codahale.metrics.Timer;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Memtable;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
+import org.apache.cassandra.io.compress.CompressionMetadata;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.utils.EstimatedHistogram;
 import org.apache.cassandra.utils.TopKSampler;
 
@@ -41,7 +42,6 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
  */
 public class TableMetrics
 {
-
     /** Total amount of data stored in the memtable that resides on-heap, including column
related overhead and partitions overwritten. */
     public final Gauge<Long> memtableOnHeapSize;
     /** Total amount of data stored in the memtable that resides off-heap, including column
related overhead and partitions overwritten. */
@@ -323,36 +323,14 @@ public class TableMetrics
         {
             public Double getValue()
             {
-                double sum = 0;
-                int total = 0;
-                for (SSTableReader sstable : cfs.getSSTables(SSTableSet.CANONICAL))
-                {
-                    if (sstable.getCompressionRatio() != MetadataCollector.NO_COMPRESSION_RATIO)
-                    {
-                        sum += sstable.getCompressionRatio();
-                        total++;
-                    }
-                }
-                return total != 0 ? sum / total : 0;
+                return computeCompressionRatio(cfs.getSSTables(SSTableSet.CANONICAL));
             }
         }, new Gauge<Double>() // global gauge
         {
             public Double getValue()
             {
-                double sum = 0;
-                int total = 0;
-                for (Keyspace keyspace : Keyspace.all())
-                {
-                    for (SSTableReader sstable : keyspace.getAllSSTables(SSTableSet.CANONICAL))
-                    {
-                        if (sstable.getCompressionRatio() != MetadataCollector.NO_COMPRESSION_RATIO)
-                        {
-                            sum += sstable.getCompressionRatio();
-                            total++;
-                        }
-                    }
-                }
-                return total != 0 ? sum / total : 0;
+                return computeCompressionRatio(Iterables.concat(Iterables.transform(Keyspace.all(),
+                                                                                    p ->
p.getAllSSTables(SSTableSet.CANONICAL))));
             }
         });
         readLatency = new LatencyMetrics(factory, "Read", cfs.keyspace.metric.readLatency,
globalReadLatency);
@@ -748,6 +726,32 @@ public class TableMetrics
     }
 
     /**
+     * Computes the compression ratio for the specified SSTables
+     *
+     * @param sstables the SSTables
+     * @return the compression ratio for the specified SSTables
+     */
+    private static Double computeCompressionRatio(Iterable<SSTableReader> sstables)
+    {
+        double compressedLengthSum = 0;
+        double dataLengthSum = 0;
+        for (SSTableReader sstable : sstables)
+        {
+            if (sstable.compression)
+            {
+                // We should not have any sstable which are in an open early mode as the
sstable were selected
+                // using SSTableSet.CANONICAL.
+                assert sstable.openReason != SSTableReader.OpenReason.EARLY;
+
+                CompressionMetadata compressionMetadata = sstable.getCompressionMetadata();
+                compressedLengthSum += compressionMetadata.compressedFileLength;
+                dataLengthSum += compressionMetadata.dataLength;
+            }
+        }
+        return dataLengthSum != 0 ? compressedLengthSum / dataLengthSum : 0;
+    }
+
+    /**
      * Create a histogram-like interface that will register both a CF, keyspace and global
level
      * histogram and forward any updates to both
      */


Mime
View raw message