cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [1/7] git commit: add snapshot space used to cfstats patch by Mikhail Stepura; reviewed by jbellis for CASSANDRA-6231
Date Tue, 26 Nov 2013 21:19:20 GMT
Updated Branches:
  refs/heads/cassandra-1.2 5f6261096 -> 3f66fbfc6
  refs/heads/cassandra-2.0 f15681b67 -> 8e8259050
  refs/heads/trunk f15d6d591 -> c1d7291c8


add snapshot space used to cfstats
patch by Mikhail Stepura; reviewed by jbellis for CASSANDRA-6231


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e178ff45
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e178ff45
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e178ff45

Branch: refs/heads/trunk
Commit: e178ff45c0510c56257c26da2dc8d082ba301522
Parents: f15d6d5
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Tue Nov 26 14:42:05 2013 -0600
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Tue Nov 26 14:49:17 2013 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |  5 ++
 .../cassandra/db/ColumnFamilyStoreMBean.java    |  5 ++
 .../org/apache/cassandra/db/Directories.java    | 90 +++++++++++++++++++-
 .../cassandra/metrics/ColumnFamilyMetrics.java  | 15 +++-
 .../org/apache/cassandra/tools/NodeCmd.java     |  1 +
 .../org/apache/cassandra/tools/NodeProbe.java   |  1 +
 7 files changed, 114 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7bb1fa0..00797e6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -18,6 +18,7 @@
 2.0.4
  * Fix setting last compacted key in the wrong level for LCS (CASSANDRA-6284)
  * Add sub-ms precision formats to the timestamp parser (CASSANDRA-6395)
+ * Add snapshot space used to cfstats (CASSANDRA-6231)
 Merged from 1.2:
  * Fix thundering herd on endpoint cache invalidation (CASSANDRA-6345)
  * cqlsh: quote single quotes in strings inside collections (CASSANDRA-6172)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 7037635..ccc15ab 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -2356,4 +2356,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         Pair<ReplayPosition, Long> truncationRecord = SystemKeyspace.getTruncationRecords().get(metadata.cfId);
         return truncationRecord == null ? Long.MIN_VALUE : truncationRecord.right;
     }
+
+    public long trueSnapshotsSize()
+    {
+        return directories.trueSnapshotsSize();
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
index 1ca922b..fc1a7b1 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
@@ -344,4 +344,9 @@ public interface ColumnFamilyStoreMBean
      * @return ratio
      */
     public double getDroppableTombstoneRatio();
+
+    /**
+     * @return the size of SSTables in "snapshots" subdirectory which aren't live anymore
+     */
+    public long trueSnapshotsSize();
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index 9795a27..ea5c2f4 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -17,16 +17,25 @@
  */
 package org.apache.cassandra.db;
 
+import static com.google.common.collect.Sets.newHashSet;
+
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOError;
 import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
 import com.google.common.primitives.Longs;
 import com.google.common.util.concurrent.Uninterruptibles;
 
@@ -35,7 +44,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.*;
-import org.apache.cassandra.db.compaction.LeveledManifest;
 import org.apache.cassandra.io.FSError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.util.FileUtils;
@@ -332,7 +340,7 @@ public class Directories
         private FileFilter getFilter()
         {
             // Note: the prefix needs to include cfname + separator to distinguish between
a cfs and it's secondary indexes
-            final String sstablePrefix = keyspacename + Component.separator + cfname + Component.separator;
+            final String sstablePrefix = getSSTablePrefix();
             return new FileFilter()
             {
                 // This function always return false since accepts adds to the components
map
@@ -401,6 +409,37 @@ public class Directories
         }
         throw new RuntimeException("Snapshot " + snapshotName + " doesn't exist");
     }
+    
+    public long trueSnapshotsSize()
+    {
+        long result = 0L;
+        for (File dir : sstableDirectories)
+            result += getTrueAllocatedSizeIn(new File(dir, join(SNAPSHOT_SUBDIR)));
+        return result;
+    }
+
+    private String getSSTablePrefix()
+    {
+        return keyspacename + Component.separator + cfname + Component.separator;
+    }
+
+    public long getTrueAllocatedSizeIn(File input)
+    {
+        if (!input.isDirectory())
+            return 0;
+        
+        TrueFilesSizeVisitor visitor = new TrueFilesSizeVisitor();
+        try
+        {
+            Files.walkFileTree(input.toPath(), visitor);
+        }
+        catch (IOException e)
+        {
+            logger.error("Could not calculate the size of {}. {}", input, e);
+        }
+    
+        return visitor.getAllocatedSize();
+    }
 
     private static File getOrCreate(File base, String... subdirs)
     {
@@ -436,4 +475,51 @@ public class Directories
         for (int i = 0; i < locations.length; ++i)
             dataFileLocations[i] = new DataDirectory(new File(locations[i]));
     }
+    
+    private class TrueFilesSizeVisitor extends SimpleFileVisitor<Path>
+    {
+        private final AtomicLong size = new AtomicLong(0);
+        private final Set<String> visited = newHashSet(); //count each file only once
+        private final Set<String> alive;
+        private final String prefix = getSSTablePrefix();
+
+        public TrueFilesSizeVisitor()
+        {
+            super();
+            Builder<String> builder = ImmutableSet.builder();
+            for (File file: sstableLister().listFiles())
+                builder.add(file.getName());
+            alive = builder.build();
+        }
+
+        private boolean isAcceptable(Path file)
+        {
+            String fileName = file.toFile().getName(); 
+            return fileName.startsWith(prefix)
+                    && !visited.contains(fileName)
+                    && !alive.contains(fileName);
+        }
+
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException
+        {
+            if (isAcceptable(file))
+            {
+                size.addAndGet(attrs.size());
+                visited.add(file.toFile().getName());
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException

+        {
+            return FileVisitResult.CONTINUE;
+        }
+        
+        public long getAllocatedSize()
+        {
+            return size.get();
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
index aabf373..038e27e 100644
--- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
+++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
@@ -82,6 +82,8 @@ public class ColumnFamilyMetrics
     public final Histogram tombstoneScannedHistogram;
     /** Live cells scanned in queries on this CF */
     public final Histogram liveScannedHistogram;
+    /** Disk space used by snapshot files which */
+    public final Gauge<Long> trueSnapshotsSize;
 
     public final Timer coordinatorReadLatency;
     public final Timer coordinatorScanLatency;
@@ -238,7 +240,7 @@ public class ColumnFamilyMetrics
             public Long value()
             {
                 long count = 0L;
-                for (SSTableReader sstable: cfs.getSSTables())
+                for (SSTableReader sstable : cfs.getSSTables())
                     count += sstable.getRecentBloomFilterFalsePositiveCount();
                 return count;
             }
@@ -249,7 +251,7 @@ public class ColumnFamilyMetrics
             {
                 long falseCount = 0L;
                 long trueCount = 0L;
-                for (SSTableReader sstable: cfs.getSSTables())
+                for (SSTableReader sstable : cfs.getSSTables())
                 {
                     falseCount += sstable.getBloomFilterFalsePositiveCount();
                     trueCount += sstable.getBloomFilterTruePositiveCount();
@@ -308,6 +310,14 @@ public class ColumnFamilyMetrics
         liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"),
true);
         coordinatorReadLatency = Metrics.newTimer(factory.createMetricName("CoordinatorReadLatency"),
TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
         coordinatorScanLatency = Metrics.newTimer(factory.createMetricName("CoordinatorScanLatency"),
TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
+        
+        trueSnapshotsSize = Metrics.newGauge(factory.createMetricName("SnapshotsSize"), new
Gauge<Long>()
+        {
+            public Long value()
+            {
+                return cfs.trueSnapshotsSize();
+            }
+        });
     }
 
     public void updateSSTableIterated(int count)
@@ -349,6 +359,7 @@ public class ColumnFamilyMetrics
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorReadLatency"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorScanLatency"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("SnapshotsSize"));
     }
 
     class ColumnFamilyMetricNameFactory implements MetricNameFactory

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/tools/NodeCmd.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java
index d415f06..a6c0010 100644
--- a/src/java/org/apache/cassandra/tools/NodeCmd.java
+++ b/src/java/org/apache/cassandra/tools/NodeCmd.java
@@ -885,6 +885,7 @@ public class NodeCmd
                 }
                 outs.println("\t\tSpace used (live), bytes: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "LiveDiskSpaceUsed"));
                 outs.println("\t\tSpace used (total), bytes: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "TotalDiskSpaceUsed"));
+                outs.println("\t\tSpace used by snapshots (total), bytes: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "SnapshotsSize"));
                 outs.println("\t\tSSTable Compression Ratio: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "CompressionRatio"));
                 outs.println("\t\tMemtable cell count: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "MemtableColumnsCount"));
                 outs.println("\t\tMemtable data size, bytes: " + probe.getColumnFamilyMetric(keyspaceName,
cfName, "MemtableDataSize"));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index 48d2013..2489de6 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -935,6 +935,7 @@ public class NodeProbe
                 case "PendingTasks":
                 case "RecentBloomFilterFalsePositives":
                 case "RecentBloomFilterFalseRatio":
+                case "SnapshotsSize":
                     return JMX.newMBeanProxy(mbeanServerConn,
                             new ObjectName("org.apache.cassandra.metrics:type=ColumnFamily,keyspace="
+ ks + ",scope=" + cf + ",name=" + metricName),
                             JmxReporter.GaugeMBean.class).getValue();


Mime
View raw message