cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [1/2] cassandra git commit: Do more aggressive ttl expiration checks to be able to drop more sstables
Date Fri, 14 Nov 2014 10:11:29 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 3b6edc6af -> 41a35ec74


Do more aggressive ttl expiration checks to be able to drop more sstables

Patch by Bjorn Hegerfors; reviewed by marcuse and slebresne for CASSANDRA-8243


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/abbcfc5f
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/abbcfc5f
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/abbcfc5f

Branch: refs/heads/trunk
Commit: abbcfc5f72323d0c6040a3cc02aba8f2c0058d95
Parents: 054beee
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Fri Nov 14 10:54:39 2014 +0100
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Fri Nov 14 11:02:45 2014 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../db/compaction/CompactionController.java     | 20 ++++----
 .../cassandra/db/compaction/TTLExpiryTest.java  | 50 ++++++++++++++++++++
 3 files changed, 60 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6228893..2476d25 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.3
+ * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243)
  * Add more log info if readMeter is null (CASSANDRA-8238)
  * add check of the system wall clock time at startup (CASSANDRA-8305)
  * Support for frozen collections (CASSANDRA-7859)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index ef27805..f23d39a 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -92,12 +92,11 @@ public class CompactionController implements AutoCloseable
      * Finds expired sstables
      *
      * works something like this;
-     * 1. find "global" minTimestamp of overlapping sstables (excluding the possibly droppable
ones)
-     * 2. build a list of candidates to be dropped
-     * 3. sort the candidate list, biggest maxTimestamp first in list
-     * 4. check if the candidates to be dropped actually can be dropped (maxTimestamp <
global minTimestamp) and it is included in the compaction
-     *    - if not droppable, update global minTimestamp and remove from candidates
-     * 5. return candidates.
+     * 1. find "global" minTimestamp of overlapping sstables and compacting sstables containing
any non-expired data
+     * 2. build a list of fully expired candidates
+     * 3. check if the candidates to be dropped actually can be dropped (maxTimestamp <
global minTimestamp)
+     *    - if not droppable, remove from candidates
+     * 4. return candidates.
      *
      * @param cfStore
      * @param compacting we take the drop-candidates from this set, it is usually the sstables
included in the compaction
@@ -127,10 +126,10 @@ public class CompactionController implements AutoCloseable
                 minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp());
         }
 
-        // we still need to keep candidates that might shadow something in a
-        // non-candidate sstable. And if we remove a sstable from the candidates, we
-        // must take it's timestamp into account (hence the sorting below).
-        Collections.sort(candidates, SSTableReader.maxTimestampComparator);
+        // At this point, minTimestamp denotes the lowest timestamp of any relevant
+        // SSTable that contains a constructive value. candidates contains all the
+        // candidates with no constructive values. The ones out of these that have
+        // (getMaxTimestamp() < minTimestamp) serve no purpose anymore.
 
         Iterator<SSTableReader> iterator = candidates.iterator();
         while (iterator.hasNext())
@@ -138,7 +137,6 @@ public class CompactionController implements AutoCloseable
             SSTableReader candidate = iterator.next();
             if (candidate.getMaxTimestamp() >= minTimestamp)
             {
-                minTimestamp = Math.min(candidate.getMinTimestamp(), minTimestamp);
                 iterator.remove();
             }
             else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/abbcfc5f/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index b98af68..4fe5cfb 100644
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction;
  * 
  */
 
+import com.google.common.collect.Sets;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
@@ -31,6 +32,10 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.io.sstable.SSTableReader;
 import org.apache.cassandra.io.sstable.SSTableScanner;
 import org.apache.cassandra.utils.ByteBufferUtil;
+
+import java.util.Collections;
+import java.util.Set;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -38,6 +43,51 @@ import static org.junit.Assert.assertTrue;
 public class TTLExpiryTest extends SchemaLoader
 {
     @Test
+    public void testAggressiveFullyExpired()
+    {
+        ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+        cfs.disableAutoCompaction();
+        cfs.metadata.gcGraceSeconds(0);
+
+        DecoratedKey ttlKey = Util.dk("ttl");
+        Mutation rm = new Mutation("Keyspace1", ttlKey.getKey());
+        rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 1, 1);
+        rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 3, 1);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        rm = new Mutation("Keyspace1", ttlKey.getKey());
+        rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 2, 1);
+        rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 5, 1);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        rm = new Mutation("Keyspace1", ttlKey.getKey());
+        rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 4, 1);
+        rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 7,
1);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        rm = new Mutation("Keyspace1", ttlKey.getKey());
+        rm.add("Standard1", Util.cellname("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 6,
3);
+        rm.add("Standard1", Util.cellname("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 8, 1);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
+        int now = (int)(System.currentTimeMillis() / 1000);
+        int gcBefore = now + 2;
+        Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
+                cfs,
+                sstables,
+                Collections.EMPTY_SET,
+                gcBefore);
+        assertEquals(2, expired.size());
+
+        cfs.clearUnsafe();
+    }
+
+    @Test
     public void testSimpleExpire() throws InterruptedException
     {
         ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");


Mime
View raw message