cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [04/15] cassandra git commit: Add ability to disable purgable tombstone check via startup flag
Date Thu, 26 May 2016 08:03:34 GMT
Add ability to disable purgable tombstone check via startup flag

Patch by marcuse; reviewed by Paulo Motta for CASSANDRA-11831


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5dc74149
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5dc74149
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5dc74149

Branch: refs/heads/cassandra-3.0
Commit: 5dc741497046c5c5b92ad2441db3865b90dc035e
Parents: 0318046
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Fri May 20 10:37:34 2016 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Thu May 26 08:49:00 2016 +0200

----------------------------------------------------------------------
 .../compaction/AbstractCompactionStrategy.java  |   2 +-
 .../db/compaction/CompactionController.java     |  26 +++-
 .../cassandra/db/compaction/NeverPurgeTest.java | 125 +++++++++++++++++++
 3 files changed, 149 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5dc74149/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 77ca404..bcd37ac 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -359,7 +359,7 @@ public abstract class AbstractCompactionStrategy
      */
     protected boolean worthDroppingTombstones(SSTableReader sstable, int gcBefore)
     {
-        if (disableTombstoneCompactions)
+        if (disableTombstoneCompactions || CompactionController.NEVER_PURGE_TOMBSTONES)
             return false;
         // since we use estimations to calculate, there is a chance that compaction will
not drop tombstones actually.
         // if that happens we will end up in infinite compaction loop, so first we check
enough if enough time has

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5dc74149/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionController.java b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
index 35d0832..1139ee4 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@ -41,8 +41,13 @@ import org.apache.cassandra.utils.concurrent.Refs;
 public class CompactionController implements AutoCloseable
 {
     private static final Logger logger = LoggerFactory.getLogger(CompactionController.class);
+    static final boolean NEVER_PURGE_TOMBSTONES = Boolean.getBoolean("cassandra.never_purge_tombstones");
 
     public final ColumnFamilyStore cfs;
+
+    // note that overlappingTree and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES
is set - this is a
+    // good thing so that noone starts using them and thinks that if overlappingSSTables
is empty, there
+    // is no overlap.
     private DataTracker.SSTableIntervalTree overlappingTree;
     private Refs<SSTableReader> overlappingSSTables;
     private final Iterable<SSTableReader> compacting;
@@ -54,17 +59,25 @@ public class CompactionController implements AutoCloseable
         this(cfs, null, maxValue);
     }
 
-    public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting,
 int gcBefore)
+    public CompactionController(ColumnFamilyStore cfs, Set<SSTableReader> compacting,
int gcBefore)
     {
         assert cfs != null;
         this.cfs = cfs;
         this.gcBefore = gcBefore;
         this.compacting = compacting;
         refreshOverlaps();
+        if (NEVER_PURGE_TOMBSTONES)
+            logger.warn("You are running with -Dcassandra.never_purge_tombstones=true, this
is dangerous!");
     }
 
     void maybeRefreshOverlaps()
     {
+        if (NEVER_PURGE_TOMBSTONES)
+        {
+            logger.debug("not refreshing overlaps - running with -Dcassandra.never_purge_tombstones=true");
+            return;
+        }
+
         for (SSTableReader reader : overlappingSSTables)
         {
             if (reader.isMarkedCompacted())
@@ -77,6 +90,9 @@ public class CompactionController implements AutoCloseable
 
     private void refreshOverlaps()
     {
+        if (NEVER_PURGE_TOMBSTONES)
+            return;
+
         if (this.overlappingSSTables != null)
             overlappingSSTables.release();
 
@@ -112,7 +128,7 @@ public class CompactionController implements AutoCloseable
     {
         logger.debug("Checking droppable sstables in {}", cfStore);
 
-        if (compacting == null)
+        if (compacting == null || NEVER_PURGE_TOMBSTONES)
             return Collections.<SSTableReader>emptySet();
 
         List<SSTableReader> candidates = new ArrayList<SSTableReader>();
@@ -175,6 +191,9 @@ public class CompactionController implements AutoCloseable
      */
     public long maxPurgeableTimestamp(DecoratedKey key)
     {
+        if (NEVER_PURGE_TOMBSTONES)
+            return Long.MIN_VALUE;
+
         List<SSTableReader> filteredSSTables = overlappingTree.search(key);
         long min = Long.MAX_VALUE;
         for (SSTableReader sstable : filteredSSTables)
@@ -191,6 +210,7 @@ public class CompactionController implements AutoCloseable
 
     public void close()
     {
-        overlappingSSTables.release();
+        if (overlappingSSTables != null)
+            overlappingSSTables.release();
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5dc74149/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
new file mode 100644
index 0000000..41db92d
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collection;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DeletedCell;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.OnDiskAtom;
+import org.apache.cassandra.db.RangeTombstone;
+import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.SSTableReader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NeverPurgeTest extends CQLTester
+{
+    static
+    {
+        System.setProperty("cassandra.never_purge_tombstones", "true");
+    }
+
+    @Test
+    public void neverPurgeCellTombstoneTest() throws Throwable
+    {
+        testHelper("UPDATE %s SET c = null WHERE a=1 AND b=2");
+    }
+
+    @Test
+    public void neverPurgeRangeTombstoneTest() throws Throwable
+    {
+        testHelper("DELETE FROM %s WHERE a=1 AND b=2");
+    }
+
+    @Test
+    public void neverPurgePartitionTombstoneTest() throws Throwable
+    {
+        testHelper("DELETE FROM %s WHERE a=1");
+    }
+
+    @Test
+    public void minorNeverPurgeTombstonesTest() throws Throwable
+    {
+        createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds
= 0");
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        cfs.disableAutoCompaction();
+        for (int i = 0; i < 4; i++)
+        {
+            for (int j = 0; j < 1000; j++)
+            {
+                execute("INSERT INTO %s (a, b, c) VALUES (" + j + ", 2, '3')");
+            }
+            cfs.forceBlockingFlush();
+        }
+
+        execute("UPDATE %s SET c = null WHERE a=1 AND b=2");
+        execute("DELETE FROM %s WHERE a=2 AND b=2");
+        execute("DELETE FROM %s WHERE a=3");
+        cfs.forceBlockingFlush();
+        cfs.enableAutoCompaction();
+        while (cfs.getSSTables().size() > 1)
+            Thread.sleep(100);
+        verifyContainsTombstones(cfs.getSSTables(), 3);
+    }
+
+    private void testHelper(String deletionStatement) throws Throwable
+    {
+        createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds
= 0");
+        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+        execute("INSERT INTO %s (a, b, c) VALUES (1, 2, '3')");
+        execute(deletionStatement);
+        Thread.sleep(1000);
+        cfs.forceBlockingFlush();
+        cfs.forceMajorCompaction();
+        verifyContainsTombstones(cfs.getSSTables(), 1);
+    }
+
+    private void verifyContainsTombstones(Collection<SSTableReader> sstables, int expectedTombstoneCount)
throws Exception
+    {
+        assertTrue(sstables.size() == 1); // always run a major compaction before calling
this
+        SSTableReader sstable = sstables.iterator().next();
+        int tombstoneCount = 0;
+        try (ISSTableScanner scanner = sstable.getScanner())
+        {
+            while (scanner.hasNext())
+            {
+                try (OnDiskAtomIterator iter = scanner.next())
+                {
+                    if (iter.getColumnFamily().deletionInfo().getTopLevelDeletion().localDeletionTime
< Integer.MAX_VALUE)
+                        tombstoneCount++;
+
+                    while (iter.hasNext())
+                    {
+                        OnDiskAtom atom = iter.next();
+                        if (atom instanceof DeletedCell || atom instanceof RangeTombstone)
+                            tombstoneCount++;
+                    }
+                }
+            }
+        }
+        assertEquals(tombstoneCount, expectedTombstoneCount);
+    }
+}


Mime
View raw message