cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1205475 - in /cassandra/branches/cassandra-1.0: ./ contrib/ interface/thrift/gen-java/org/apache/cassandra/thrift/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/compaction/ test/unit/org/apache/cassandra/ test/unit/org...
Date Wed, 23 Nov 2011 16:18:45 GMT
Author: jbellis
Date: Wed Nov 23 16:18:43 2011
New Revision: 1205475

URL: http://svn.apache.org/viewvc?rev=1205475&view=rev
Log:
merge #2786 from 0.8

Modified:
    cassandra/branches/cassandra-1.0/   (props changed)
    cassandra/branches/cassandra-1.0/CHANGES.txt
    cassandra/branches/cassandra-1.0/contrib/   (props changed)
    cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
  (props changed)
    cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
  (props changed)
    cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
  (props changed)
    cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
  (props changed)
    cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
  (props changed)
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/EchoedRow.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
    cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/SchemaLoader.java
    cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java

Propchange: cassandra/branches/cassandra-1.0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7:1026516-1183000
 /cassandra/branches/cassandra-0.7.0:1053690-1055654
-/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0:1125021-1130369
 /cassandra/branches/cassandra-0.8.1:1101014-1125018
 /cassandra/branches/cassandra-1.0:1167106,1167185

Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Wed Nov 23 16:18:43 2011
@@ -10,6 +10,10 @@
 Merged from 0.8:
  * fix concurrence issue in the FailureDetector (CASSANDRA-3519)
  * fix array out of bounds error in counter shard removal (CASSANDRA-3514)
+Merged from 0.8:
+ * avoid dropping tombstones when they might still be needed to shadow
+   data in a different sstable (CASSANDRA-2786)
+
 
 1.0.3
  * revert name-based query defragmentation aka CASSANDRA-2503 (CASSANDRA-3491)

Propchange: cassandra/branches/cassandra-1.0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/contrib:922689-1052356,1052358-1053452,1053454,1053456-1068009
 /cassandra/branches/cassandra-0.7/contrib:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/contrib:1053690-1055654
-/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/contrib:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/contrib:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/contrib:1101014-1125018
 /cassandra/branches/cassandra-1.0/contrib:1167106,1167185

Propchange: cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1101014-1125018
 /cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java:1167106,1167185

Propchange: cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1101014-1125018
 /cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java:1167106,1167185

Propchange: cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1101014-1125018
 /cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/InvalidRequestException.java:1167106,1167185

Propchange: cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1101014-1125018
 /cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/NotFoundException.java:1167106,1167185

Propchange: cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 23 16:18:43 2011
@@ -1,7 +1,7 @@
 /cassandra/branches/cassandra-0.6/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:922689-1052356,1052358-1053452,1053454,1053456-1131291
 /cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1026516-1183000
 /cassandra/branches/cassandra-0.7.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1053690-1055654
-/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1205232,1205316
+/cassandra/branches/cassandra-0.8/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1090934-1125013,1125019-1205453
 /cassandra/branches/cassandra-0.8.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1125021-1130369
 /cassandra/branches/cassandra-0.8.1/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1101014-1125018
 /cassandra/branches/cassandra-1.0/interface/thrift/gen-java/org/apache/cassandra/thrift/SuperColumn.java:1167106,1167185

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/EchoedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/EchoedRow.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/EchoedRow.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/EchoedRow.java Wed Nov
23 16:18:43 2011
@@ -36,13 +36,11 @@ import org.apache.cassandra.io.sstable.S
 public class EchoedRow extends AbstractCompactedRow
 {
     private final SSTableIdentityIterator row;
-    private final int gcBefore;
 
-    public EchoedRow(CompactionController controller, SSTableIdentityIterator row)
+    public EchoedRow(SSTableIdentityIterator row)
     {
         super(row.getKey());
         this.row = row;
-        this.gcBefore = controller.gcBefore;
         // Reset SSTableIdentityIterator because we have not guarantee the filePointer hasn't
moved since the Iterator was built
         row.reset();
     }
@@ -63,7 +61,8 @@ public class EchoedRow extends AbstractC
 
     public boolean isEmpty()
     {
-        return !row.hasNext() && ColumnFamilyStore.removeDeletedCF(row.getColumnFamily(),
gcBefore) == null;
+        // never okay to purge a EchoedRow -- if it were, we'd need to deserialize instead
of echoing
+        return false;
     }
 
     public int columnCount()

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionController.java
Wed Nov 23 16:18:43 2011
@@ -72,6 +72,10 @@ public class CompactionController
         return cfs.columnFamily;
     }
 
+    /**
+     * @return true if it's okay to drop tombstones for the given row, i.e., if we know all
the verisons of the row
+     * are included in the compaction set
+     */
     public boolean shouldPurge(DecoratedKey key)
     {
         return !cfs.isKeyInRemainingSSTables(key, sstables);
@@ -120,7 +124,7 @@ public class CompactionController
             && (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit() || !keyExistenceIsExpensive)
             && !shouldPurge(rows.get(0).getKey()))
         {
-            return new EchoedRow(this, rows.get(0));
+            return new EchoedRow(rows.get(0));
         }
 
         if (rowSize > DatabaseDescriptor.getInMemoryCompactionLimit())

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/LazilyCompactedRow.java
Wed Nov 23 16:18:43 2011
@@ -156,7 +156,9 @@ public class LazilyCompactedRow extends 
 
     public boolean isEmpty()
     {
-        boolean cfIrrelevant = ColumnFamilyStore.removeDeletedCF(emptyColumnFamily, controller.gcBefore)
== null;
+        boolean cfIrrelevant = shouldPurge
+                             ? ColumnFamilyStore.removeDeletedCF(emptyColumnFamily, controller.gcBefore)
== null
+                             : !emptyColumnFamily.isMarkedForDelete(); // tombstones are
relevant
         return cfIrrelevant && columnCount == 0;
     }
 

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/ParallelCompactionIterable.java
Wed Nov 23 16:18:43 2011
@@ -109,7 +109,7 @@ public class ParallelCompactionIterable 
             {
                 compactedRow = container.future == null
                              ? container.row
-                             : new PrecompactedRow(container.key, controller, container.future.get());
+                             : new PrecompactedRow(container.key, container.future.get());
             }
             catch (InterruptedException e)
             {

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java
Wed Nov 23 16:18:43 2011
@@ -46,21 +46,11 @@ public class PrecompactedRow extends Abs
     private static Logger logger = LoggerFactory.getLogger(PrecompactedRow.class);
 
     private final ColumnFamily compactedCf;
-    private final int gcBefore;
-
-    // For testing purposes
-    public PrecompactedRow(DecoratedKey<?> key, ColumnFamily compacted)
-    {
-        super(key);
-        this.compactedCf = compacted;
-        this.gcBefore = Integer.MAX_VALUE;
-    }
 
     /** it is caller's responsibility to call removeDeleted + removeOldShards from the cf
before calling this constructor */
-    public PrecompactedRow(DecoratedKey<?> key, CompactionController controller, ColumnFamily
cf)
+    public PrecompactedRow(DecoratedKey<?> key, ColumnFamily cf)
     {
         super(key);
-        this.gcBefore = controller.gcBefore;
         compactedCf = cf;
     }
 
@@ -100,7 +90,6 @@ public class PrecompactedRow extends Abs
     public PrecompactedRow(CompactionController controller, List<SSTableIdentityIterator>
rows)
     {
         this(rows.get(0).getKey(),
-             controller,
              removeDeletedAndOldShards(rows.get(0).getKey(), controller, merge(rows)));
     }
 
@@ -165,7 +154,7 @@ public class PrecompactedRow extends Abs
 
     public boolean isEmpty()
     {
-        return compactedCf == null || ColumnFamilyStore.removeDeletedCF(compactedCf, gcBefore)
== null;
+        return compactedCf == null;
     }
 
     public int columnCount()

Modified: cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/SchemaLoader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/SchemaLoader.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/SchemaLoader.java (original)
+++ cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/SchemaLoader.java Wed
Nov 23 16:18:43 2011
@@ -156,6 +156,7 @@ public class SchemaLoader
                                                           bytes)
                                                    .defaultValidator(CounterColumnType.instance)
                                                    .mergeShardsChance(1.0),
+                                           superCFMD(ks1, "SuperDirectGC", BytesType.instance).gcGraceSeconds(0),
                                            jdbcCFMD(ks1, "JdbcInteger", IntegerType.instance).columnMetadata(integerColumn),
                                            jdbcCFMD(ks1, "JdbcUtf8", UTF8Type.instance).columnMetadata(utf8Column),
                                            jdbcCFMD(ks1, "JdbcLong", LongType.instance),

Modified: cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java?rev=1205475&r1=1205474&r2=1205475&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
(original)
+++ cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
Wed Nov 23 16:18:43 2011
@@ -218,15 +218,18 @@ public class CompactionsTest extends Cle
     public void testDontPurgeAccidentaly() throws IOException, ExecutionException, InterruptedException
     {
         // Testing with and without forcing deserialization. Without deserialization, EchoedRow
will be used.
-        testDontPurgeAccidentaly("test1", false);
-        testDontPurgeAccidentaly("test2", true);
+        testDontPurgeAccidentaly("test1", "Super5", false);
+        testDontPurgeAccidentaly("test2", "Super5", true);
+
+        // Use CF with gc_grace=0, see last bug of CASSANDRA-2786
+        testDontPurgeAccidentaly("test1", "SuperDirectGC", false);
+        testDontPurgeAccidentaly("test2", "SuperDirectGC", true);
     }
 
-    private void testDontPurgeAccidentaly(String k, boolean forceDeserialize) throws IOException,
ExecutionException, InterruptedException
+    private void testDontPurgeAccidentaly(String k, String cfname, boolean forceDeserialize)
throws IOException, ExecutionException, InterruptedException
     {
         // This test catches the regression of CASSANDRA-2786
         Table table = Table.open(TABLE1);
-        String cfname = "Super5";
         ColumnFamilyStore store = table.getColumnFamilyStore(cfname);
 
         // disable compaction while flushing
@@ -252,7 +255,7 @@ public class CompactionsTest extends Cle
         rm.apply();
 
         ColumnFamily cf = store.getColumnFamily(filter);
-        assert cf.isEmpty() : "should be empty: " + cf;
+        assert cf == null || cf.isEmpty() : "should be empty: " + cf;
 
         store.forceBlockingFlush();
 
@@ -265,6 +268,6 @@ public class CompactionsTest extends Cle
         Util.compact(store, toCompact, forceDeserialize);
 
         cf = store.getColumnFamily(filter);
-        assert cf.isEmpty() : "should be empty: " + cf;
+        assert cf == null || cf.isEmpty() : "should be empty: " + cf;
     }
 }



Mime
View raw message