cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1205452 - in /cassandra/branches/cassandra-0.8: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ test/unit/org/apache/cassandra/ test/unit/org/apache/cassandra/db/compaction/
Date Wed, 23 Nov 2011 15:38:47 GMT
Author: jbellis
Date: Wed Nov 23 15:38:44 2011
New Revision: 1205452

URL: http://svn.apache.org/viewvc?rev=1205452&view=rev
Log:
avoid dropping tombstones when they might still be needed to shadow data in another sstable
patch by slebresne and jbellis for CASSANDRA-2786

Modified:
    cassandra/branches/cassandra-0.8/CHANGES.txt
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/EchoedRow.java
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/CompactionController.java
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
    cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
    cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/SchemaLoader.java
    cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java

Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Wed Nov 23 15:38:44 2011
@@ -1,4 +1,6 @@
 0.8.8
+ * avoid dropping tombstones when they might still be needed to shadow
+   data in a different sstable (CASSANDRA-2786)
  * fix truncate allowing data to be replayed post-restart (CASSANDRA-3297)
  * make iwriter final in IndexWriter to avoid NPE (CASSANDRA-2863)
  * (CQL) update grammar to require key clause in DELETE statement

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/EchoedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/EchoedRow.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/EchoedRow.java (original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/EchoedRow.java Wed Nov
23 15:38:44 2011
@@ -36,13 +36,11 @@ import org.apache.cassandra.io.sstable.S
 public class EchoedRow extends AbstractCompactedRow
 {
     private final SSTableIdentityIterator row;
-    private final int gcBefore;
 
-    public EchoedRow(CompactionController controller, SSTableIdentityIterator row)
+    public EchoedRow(SSTableIdentityIterator row)
     {
         super(row.getKey());
         this.row = row;
-        this.gcBefore = controller.gcBefore;
         // Reset SSTableIdentityIterator because we have not guarantee the filePointer hasn't
moved since the Iterator was built
         row.reset();
     }
@@ -62,7 +60,8 @@ public class EchoedRow extends AbstractC
 
     public boolean isEmpty()
     {
-        return !row.hasNext() && ColumnFamilyStore.removeDeletedCF(row.getColumnFamily(),
gcBefore) == null;
+        // never okay to purge a EchoedRow -- if it were, we'd need to deserialize instead
of echoing
+        return false;
     }
 
     public int columnCount()

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/CompactionController.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/CompactionController.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/CompactionController.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/CompactionController.java
Wed Nov 23 15:38:44 2011
@@ -31,7 +31,6 @@ import org.apache.cassandra.db.Decorated
 import org.apache.cassandra.db.EchoedRow;
 import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
 import org.apache.cassandra.io.sstable.SSTableReader;
-import org.apache.cassandra.utils.ByteBufferUtil;
 
 /**
  * Manage compaction options.
@@ -87,6 +86,10 @@ public class CompactionController
         return cfs.columnFamily;
     }
 
+    /**
+     * @return true if it's okay to drop tombstones for the given row, i.e., if we know all
the verisons of the row
+     * are included in the compaction set
+     */
     public boolean shouldPurge(DecoratedKey key)
     {
         return isMajor || !cfs.isKeyInRemainingSSTables(key, sstables);
@@ -131,7 +134,7 @@ public class CompactionController
     public AbstractCompactedRow getCompactedRow(List<SSTableIdentityIterator> rows)
     {
         if (rows.size() == 1 && !needDeserialize() && !shouldPurge(rows.get(0).getKey()))
-            return new EchoedRow(this, rows.get(0));
+            return new EchoedRow(rows.get(0));
 
         long rowSize = 0;
         for (SSTableIdentityIterator row : rows)

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
Wed Nov 23 15:38:44 2011
@@ -146,7 +146,9 @@ public class LazilyCompactedRow extends 
 
     public boolean isEmpty()
     {
-        boolean cfIrrelevant = ColumnFamilyStore.removeDeletedCF(emptyColumnFamily, controller.gcBefore)
== null;
+        boolean cfIrrelevant = shouldPurge
+                             ? ColumnFamilyStore.removeDeletedCF(emptyColumnFamily, controller.gcBefore)
== null
+                             : !emptyColumnFamily.isMarkedForDelete(); // tombstones are
relevant
         return cfIrrelevant && columnCount == 0;
     }
 

Modified: cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
(original)
+++ cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
Wed Nov 23 15:38:44 2011
@@ -45,14 +45,12 @@ public class PrecompactedRow extends Abs
     private static Logger logger = LoggerFactory.getLogger(PrecompactedRow.class);
 
     private final ColumnFamily compactedCf;
-    private final int gcBefore;
 
     // For testing purposes
     public PrecompactedRow(DecoratedKey key, ColumnFamily compacted)
     {
         super(key);
         this.compactedCf = compacted;
-        this.gcBefore = Integer.MAX_VALUE;
     }
 
     public static ColumnFamily removeDeletedAndOldShards(DecoratedKey key, CompactionController
controller, ColumnFamily cf)
@@ -71,7 +69,6 @@ public class PrecompactedRow extends Abs
     public PrecompactedRow(CompactionController controller, List<SSTableIdentityIterator>
rows)
     {
         super(rows.get(0).getKey());
-        gcBefore = controller.gcBefore;
         compactedCf = removeDeletedAndOldShards(rows.get(0).getKey(), controller, merge(rows));
     }
 
@@ -133,7 +130,7 @@ public class PrecompactedRow extends Abs
 
     public boolean isEmpty()
     {
-        return compactedCf == null || ColumnFamilyStore.removeDeletedCF(compactedCf, gcBefore)
== null;
+        return compactedCf == null;
     }
 
     public int columnCount()

Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/SchemaLoader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/SchemaLoader.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/SchemaLoader.java (original)
+++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/SchemaLoader.java Wed
Nov 23 15:38:44 2011
@@ -145,6 +145,7 @@ public class SchemaLoader
                                                           bytes,
                                                           bytes)
                                                    .defaultValidator(CounterColumnType.instance),
+                                           superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0),
                                            jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).columnMetadata(integerColumn),
                                            jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).columnMetadata(utf8Column),
                                            jdbcCFMD(ks1, "JdbcLong", LongType.instance),

Modified: cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java?rev=1205452&r1=1205451&r2=1205452&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
(original)
+++ cassandra/branches/cassandra-0.8/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
Wed Nov 23 15:38:44 2011
@@ -213,15 +213,18 @@ public class CompactionsTest extends Cle
     public void testDontPurgeAccidentaly() throws IOException, ExecutionException, InterruptedException
     {
         // Testing with and without forcing deserialization. Without deserialization, EchoedRow
will be used.
-        testDontPurgeAccidentaly("test1", false);
-        testDontPurgeAccidentaly("test2", true);
+        testDontPurgeAccidentaly("test1", "Super5", false);
+        testDontPurgeAccidentaly("test2", "Super5", true);
+
+        // Use CF with gc_grace=0, see last bug of CASSANDRA-2786
+        testDontPurgeAccidentaly("test1", "SuperDirectGC", false);
+        testDontPurgeAccidentaly("test2", "SuperDirectGC", true);
     }
 
-    private void testDontPurgeAccidentaly(String k, boolean forceDeserialize) throws IOException,
ExecutionException, InterruptedException
+    private void testDontPurgeAccidentaly(String k, String cfname, boolean forceDeserialize)
throws IOException, ExecutionException, InterruptedException
     {
         // This test catches the regression of CASSANDRA-2786
         Table table = Table.open(TABLE1);
-        String cfname = "Super5";
         ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
 
         // disable compaction while flushing
@@ -247,7 +250,7 @@ public class CompactionsTest extends Cle
         rm.apply();
 
         ColumnFamily cf = store.getColumnFamily(filter);
-        assert cf.isEmpty() : "should be empty: " + cf;
+        assert cf == null || cf.isEmpty() : "should be empty: " + cf;
 
         store.forceBlockingFlush();
 
@@ -261,6 +264,6 @@ public class CompactionsTest extends Cle
         CompactionManager.instance.doCompactionWithoutSizeEstimation(store, toCompact, (int)
(System.currentTimeMillis() / 1000) - store.metadata.getGcGraceSeconds(), location, forceDeserialize);
 
         cf = store.getColumnFamily(filter);
-        assert cf.isEmpty() : "should be empty: " + cf;
+        assert cf == null || cf.isEmpty() : "should be empty: " + cf;
     }
 }



Mime
View raw message