cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [2/5] cassandra git commit: Calculate last compacted keys from higher level
Date Tue, 23 Aug 2016 14:50:29 GMT
Calculate last compacted keys from higher level

patch by Dikang Gu; reviewed by marcuse for CASSANDRA-6216


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/6808e755
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/6808e755
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/6808e755

Branch: refs/heads/trunk
Commit: 6808e7555762236b6222f65bf066f90f7597405e
Parents: d0fc953
Author: Dikang Gu <dikang85@gmail.com>
Authored: Sat Aug 20 00:55:22 2016 -0700
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Aug 23 16:18:39 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../compaction/LeveledCompactionStrategy.java   |  7 ++++
 .../db/compaction/LeveledManifest.java          | 36 ++++++++++++++++++++
 3 files changed, 44 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d384fb3..0d461f3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.9
+ * Calculate last compacted key on startup (CASSANDRA-6216)
  * Add schema to snapshot manifest, add USING TIMESTAMP clause to ALTER TABLE statements
(CASSANDRA-7190)
  * Fix clean interval not sent to commit log for empty memtable flush (CASSANDRA-12436)
  * Fix potential resource leak in RMIServerSocketFactoryImpl (CASSANDRA-12331)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index cd74620..65fca27 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@ -85,6 +85,13 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
         return manifest.getAllLevelSize();
     }
 
+    @Override
+    public void startup()
+    {
+        manifest.calculateLastCompactedKeys();
+        super.startup();
+    }
+
     /**
      * the only difference between background and maximal in LCS is that maximal is still
allowed
      * (by explicit user request) even when compaction is disabled.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/6808e755/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index 2bfe88f..cf3ae1a 100644
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@ -30,6 +30,7 @@ import com.google.common.collect.Sets;
 import com.google.common.primitives.Ints;
 
 import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class LeveledManifest
     @VisibleForTesting
     protected final List<SSTableReader>[] generations;
     private final PartitionPosition[] lastCompactedKeys;
+    private volatile boolean calculatedLastCompactedKeys = false;
     private final long maxSSTableSizeInBytes;
     private final SizeTieredCompactionStrategyOptions options;
     private final int [] compactionCounter;
@@ -104,9 +106,38 @@ public class LeveledManifest
         {
             manifest.repairOverlappingSSTables(i);
         }
+        manifest.calculateLastCompactedKeys();
         return manifest;
     }
 
+    /**
+     * If we want to start compaction in level n, find the newest (by modification time)
file in level n+1
+     * and use its last token for last compacted key in level n;
+     */
+    public void calculateLastCompactedKeys()
+    {
+        for (int i = 0; i < generations.length - 1; i++)
+        {
+            // this level is empty
+            if (generations[i + 1].isEmpty())
+                continue;
+
+            SSTableReader sstableWithMaxModificationTime = null;
+            long maxModificationTime = Long.MIN_VALUE;
+            for (SSTableReader ssTableReader : generations[i + 1])
+            {
+                long modificationTime = ssTableReader.getCreationTimeFor(Component.DATA);
+                if (modificationTime >= maxModificationTime)
+                {
+                    sstableWithMaxModificationTime = ssTableReader;
+                    maxModificationTime = modificationTime;
+                }
+            }
+
+            lastCompactedKeys[i] = sstableWithMaxModificationTime.last;
+        }
+    }
+
     public synchronized void add(SSTableReader reader)
     {
         int level = reader.getSSTableLevel();
@@ -307,6 +338,11 @@ public class LeveledManifest
         // This isn't a magic wand -- if you are consistently writing too fast for LCS to
keep
         // up, you're still screwed.  But if instead you have intermittent bursts of activity,
         // it can help a lot.
+        if (!calculatedLastCompactedKeys)
+        {
+            this.calculateLastCompactedKeys();
+        }
+
         for (int i = generations.length - 1; i > 0; i--)
         {
             List<SSTableReader> sstables = getLevel(i);


Mime
View raw message