cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [07/15] cassandra git commit: Merge branch 'cassandra-2.1' into cassandra-2.2
Date Thu, 26 May 2016 08:03:37 GMT
Merge branch 'cassandra-2.1' into cassandra-2.2


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/63efa077
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/63efa077
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/63efa077

Branch: refs/heads/cassandra-2.2
Commit: 63efa07758856ccf8971e2ebbe3bfef190ea2bc2
Parents: 118bea5 5dc7414
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Thu May 26 09:01:11 2016 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Thu May 26 09:10:44 2016 +0200

----------------------------------------------------------------------
 .../compaction/AbstractCompactionStrategy.java  |   2 +-
 .../db/compaction/CompactionController.java     |  25 +++-
 .../cassandra/db/compaction/NeverPurgeTest.java | 125 +++++++++++++++++++
 3 files changed, 148 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/63efa077/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/63efa077/src/java/org/apache/cassandra/db/compaction/CompactionController.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/CompactionController.java
index 00d1344,1139ee4..699bc55
--- a/src/java/org/apache/cassandra/db/compaction/CompactionController.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionController.java
@@@ -41,10 -41,15 +41,14 @@@ import static org.apache.cassandra.db.l
  public class CompactionController implements AutoCloseable
  {
      private static final Logger logger = LoggerFactory.getLogger(CompactionController.class);
+     static final boolean NEVER_PURGE_TOMBSTONES = Boolean.getBoolean("cassandra.never_purge_tombstones");
  
      public final ColumnFamilyStore cfs;
 -
 -    // note that overlappingTree and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES
is set - this is a
++    // note that overlapIterator and overlappingSSTables will be null if NEVER_PURGE_TOMBSTONES
is set - this is a
+     // good thing so that noone starts using them and thinks that if overlappingSSTables
is empty, there
+     // is no overlap.
 -    private DataTracker.SSTableIntervalTree overlappingTree;
      private Refs<SSTableReader> overlappingSSTables;
 +    private OverlapIterator<RowPosition, SSTableReader> overlapIterator;
      private final Iterable<SSTableReader> compacting;
  
      public final int gcBefore;
@@@ -110,12 -126,12 +125,12 @@@
       */
      public static Set<SSTableReader> getFullyExpiredSSTables(ColumnFamilyStore cfStore,
Iterable<SSTableReader> compacting, Iterable<SSTableReader> overlapping, int gcBefore)
      {
 -        logger.debug("Checking droppable sstables in {}", cfStore);
 +        logger.trace("Checking droppable sstables in {}", cfStore);
  
-         if (compacting == null)
+         if (compacting == null || NEVER_PURGE_TOMBSTONES)
              return Collections.<SSTableReader>emptySet();
  
 -        List<SSTableReader> candidates = new ArrayList<SSTableReader>();
 +        List<SSTableReader> candidates = new ArrayList<>();
  
          long minTimestamp = Long.MAX_VALUE;
  
@@@ -179,9 -191,12 +194,12 @@@
       */
      public long maxPurgeableTimestamp(DecoratedKey key)
      {
+         if (NEVER_PURGE_TOMBSTONES)
+             return Long.MIN_VALUE;
+ 
 -        List<SSTableReader> filteredSSTables = overlappingTree.search(key);
          long min = Long.MAX_VALUE;
 -        for (SSTableReader sstable : filteredSSTables)
 +        overlapIterator.update(key);
 +        for (SSTableReader sstable : overlapIterator.overlaps())
          {
              // if we don't have bloom filter(bf_fp_chance=1.0 or filter file is missing),
              // we check index file instead.
@@@ -202,7 -210,7 +220,8 @@@
  
      public void close()
      {
-         overlappingSSTables.release();
+         if (overlappingSSTables != null)
+             overlappingSSTables.release();
      }
 +
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/63efa077/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
index 0000000,41db92d..5211065
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
@@@ -1,0 -1,125 +1,125 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.cassandra.db.compaction;
+ 
+ import java.util.Collection;
+ 
+ import org.junit.Test;
+ 
+ import org.apache.cassandra.cql3.CQLTester;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.DeletedCell;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.db.OnDiskAtom;
+ import org.apache.cassandra.db.RangeTombstone;
+ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
+ import org.apache.cassandra.io.sstable.ISSTableScanner;
 -import org.apache.cassandra.io.sstable.SSTableReader;
++import org.apache.cassandra.io.sstable.format.SSTableReader;
+ 
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
+ 
+ public class NeverPurgeTest extends CQLTester
+ {
+     static
+     {
+         System.setProperty("cassandra.never_purge_tombstones", "true");
+     }
+ 
+     @Test
+     public void neverPurgeCellTombstoneTest() throws Throwable
+     {
+         testHelper("UPDATE %s SET c = null WHERE a=1 AND b=2");
+     }
+ 
+     @Test
+     public void neverPurgeRangeTombstoneTest() throws Throwable
+     {
+         testHelper("DELETE FROM %s WHERE a=1 AND b=2");
+     }
+ 
+     @Test
+     public void neverPurgePartitionTombstoneTest() throws Throwable
+     {
+         testHelper("DELETE FROM %s WHERE a=1");
+     }
+ 
+     @Test
+     public void minorNeverPurgeTombstonesTest() throws Throwable
+     {
+         createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds
= 0");
+         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+         cfs.disableAutoCompaction();
+         for (int i = 0; i < 4; i++)
+         {
+             for (int j = 0; j < 1000; j++)
+             {
+                 execute("INSERT INTO %s (a, b, c) VALUES (" + j + ", 2, '3')");
+             }
+             cfs.forceBlockingFlush();
+         }
+ 
+         execute("UPDATE %s SET c = null WHERE a=1 AND b=2");
+         execute("DELETE FROM %s WHERE a=2 AND b=2");
+         execute("DELETE FROM %s WHERE a=3");
+         cfs.forceBlockingFlush();
+         cfs.enableAutoCompaction();
+         while (cfs.getSSTables().size() > 1)
+             Thread.sleep(100);
+         verifyContainsTombstones(cfs.getSSTables(), 3);
+     }
+ 
+     private void testHelper(String deletionStatement) throws Throwable
+     {
+         createTable("CREATE TABLE %s (a int, b int, c text, PRIMARY KEY (a, b)) WITH gc_grace_seconds
= 0");
+         ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
+         execute("INSERT INTO %s (a, b, c) VALUES (1, 2, '3')");
+         execute(deletionStatement);
+         Thread.sleep(1000);
+         cfs.forceBlockingFlush();
+         cfs.forceMajorCompaction();
+         verifyContainsTombstones(cfs.getSSTables(), 1);
+     }
+ 
+     private void verifyContainsTombstones(Collection<SSTableReader> sstables, int
expectedTombstoneCount) throws Exception
+     {
+         assertTrue(sstables.size() == 1); // always run a major compaction before calling
this
+         SSTableReader sstable = sstables.iterator().next();
+         int tombstoneCount = 0;
+         try (ISSTableScanner scanner = sstable.getScanner())
+         {
+             while (scanner.hasNext())
+             {
+                 try (OnDiskAtomIterator iter = scanner.next())
+                 {
+                     if (iter.getColumnFamily().deletionInfo().getTopLevelDeletion().localDeletionTime
< Integer.MAX_VALUE)
+                         tombstoneCount++;
+ 
+                     while (iter.hasNext())
+                     {
+                         OnDiskAtom atom = iter.next();
+                         if (atom instanceof DeletedCell || atom instanceof RangeTombstone)
+                             tombstoneCount++;
+                     }
+                 }
+             }
+         }
+         assertEquals(tombstoneCount, expectedTombstoneCount);
+     }
+ }


Mime
View raw message