cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1133873 - in /cassandra/branches/cassandra-0.7: ./ src/java/org/apache/cassandra/service/ test/unit/org/apache/cassandra/ test/unit/org/apache/cassandra/db/ test/unit/org/apache/cassandra/service/
Date Thu, 09 Jun 2011 13:50:27 GMT
Author: jbellis
Date: Thu Jun  9 13:50:27 2011
New Revision: 1133873

URL: http://svn.apache.org/viewvc?rev=1133873&view=rev
Log:
fix removing columns and subcolumns that are supressed by a row orsupercolumn tombstone during
replica resolution
patch by Aaron Morton and jbellis; reviewed by slebresne for CASSANDRA-2590

Modified:
    cassandra/branches/cassandra-0.7/CHANGES.txt
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/service/RowRepairResolver.java
    cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/Util.java
    cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/TableTest.java
    cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/service/RowResolverTest.java

Modified: cassandra/branches/cassandra-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1133873&r1=1133872&r2=1133873&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7/CHANGES.txt Thu Jun  9 13:50:27 2011
@@ -16,6 +16,8 @@
  * workaround large resultsets causing large allocation retention
    by nio sockets (CASSANDRA-2654)
  * fix nodetool ring use with Ec2Snitch (CASSANDRA-2733)
+ * fix removing columns and subcolumns that are supressed by a row or
+   supercolumn tombstone during replica resolution (CASSANDRA-2590)
 
 
 0.7.6

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/service/RowRepairResolver.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/service/RowRepairResolver.java?rev=1133873&r1=1133872&r2=1133873&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/service/RowRepairResolver.java
(original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/service/RowRepairResolver.java
Thu Jun  9 13:50:27 2011
@@ -22,13 +22,17 @@ import java.io.IOError;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+
+import org.apache.commons.collections.iterators.CollatingIterator;
 
 import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.filter.QueryFilter;
+import org.apache.cassandra.db.filter.QueryPath;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
+import org.apache.cassandra.utils.FBUtilities;
 
 public class RowRepairResolver extends AbstractRowResolver
 {
@@ -121,19 +125,30 @@ public class RowRepairResolver extends A
         ColumnFamily resolved = null;
         for (ColumnFamily cf : versions)
         {
-            if (cf != null)
-            {
-                resolved = cf.cloneMe();
-                break;
-            }
+            if (cf == null)
+                continue;
+
+            if (resolved == null)
+                resolved = cf.cloneMeShallow();
+            else
+                resolved.delete(cf);
         }
         if (resolved == null)
             return null;
 
-        for (ColumnFamily cf : versions)
-            resolved.resolve(cf);
-
-        return resolved;
+        // mimic the collectCollatedColumn + removeDeleted path that getColumnFamily takes.
+        // this will handle removing columns and subcolumns that are supressed by a row or
+        // supercolumn tombstone.
+        QueryFilter filter = new QueryFilter(null, new QueryPath(resolved.metadata().cfName),
new IdentityQueryFilter());
+        CollatingIterator iter = new CollatingIterator(resolved.metadata().comparator.columnComparator);
+        for (ColumnFamily version : versions)
+        {
+            if (version == null)
+                continue;
+            iter.addIterator(version.getColumnsMap().values().iterator());
+        }
+        filter.collectCollatedColumns(resolved, iter, Integer.MIN_VALUE);
+        return ColumnFamilyStore.removeDeleted(resolved, Integer.MIN_VALUE);
     }
 
     public Row getData() throws IOException

Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/Util.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/Util.java?rev=1133873&r1=1133872&r2=1133873&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/Util.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/Util.java Thu Jun  9 13:50:27
2011
@@ -51,6 +51,14 @@ public class Util
         return new Column(ByteBufferUtil.bytes(name), ByteBufferUtil.bytes(value), timestamp);
     }
 
+    public static SuperColumn superColumn(ColumnFamily cf, String name, Column... columns)
+    {
+        SuperColumn sc = new SuperColumn(ByteBufferUtil.bytes(name), cf.metadata().comparator);
+        for (Column c : columns)
+            sc.addColumn(c);
+        return sc;
+    }
+
     public static Token token(String key)
     {
         return StorageService.getPartitioner().getToken(ByteBufferUtil.bytes(key));

Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/TableTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/TableTest.java?rev=1133873&r1=1133872&r2=1133873&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/TableTest.java (original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/TableTest.java Thu
Jun  9 13:50:27 2011
@@ -510,7 +510,21 @@ public class TableTest extends CleanupHe
 
     public static void assertColumns(ColumnFamily cf, String... columnNames)
     {
-        Collection<IColumn> columns = cf == null ? new TreeSet<IColumn>() : cf.getSortedColumns();
+        assertColumns((IColumnContainer)cf, columnNames);
+    }
+
+    public static void assertSubColumns(ColumnFamily cf, String scName, String... columnNames)
+    {
+        IColumnContainer sc = cf == null ? null : ((IColumnContainer)cf.getColumn(ByteBufferUtil.bytes(scName)));
+        assertColumns(sc, columnNames);
+    }
+
+    public static void assertColumns(IColumnContainer container, String... columnNames)
+    {
+        Collection<IColumn> columns = container == null
+                                      ? new TreeSet<IColumn>()
+                                      : (container instanceof ColumnFamily) ? ((ColumnFamily)
container).getSortedColumns()
+                                      : ((SuperColumn) container).getSubColumns();
         List<String> L = new ArrayList<String>();
         for (IColumn column : columns)
         {
@@ -539,9 +553,28 @@ public class TableTest extends CleanupHe
 
         assert Arrays.equals(la, columnNames1)
                 : String.format("Columns [%s(as string: %s)])] is not expected [%s]",
-                                ((cf == null) ? "" : cf.getComparator().getColumnsString(columns)),
+                                ((container == null) ? "" : container.getComparator().getColumnsString(columns)),
                                 lasb.toString(),
                                 StringUtils.join(columnNames1, ","));
     }
 
+    public static void assertColumn(ColumnFamily cf, String name, String value, long timestamp)
+    {
+        assertColumn(cf.getColumn(ByteBufferUtil.bytes(name)), value, timestamp);
+    }
+
+    public static void assertSubColumn(ColumnFamily cf, String scName, String name, String
value, long timestamp)
+    {
+        SuperColumn sc = (SuperColumn)cf.getColumn(ByteBufferUtil.bytes(scName));
+        assertColumn(sc.getSubColumn(ByteBufferUtil.bytes(name)), value, timestamp);
+    }
+
+    public static void assertColumn(IColumn column, String value, long timestamp)
+    {
+        assertNotNull(column);
+        assertEquals(0, ByteBufferUtil.compareUnsigned(column.value(), ByteBufferUtil.bytes(value)));
+        assertEquals(timestamp, column.timestamp());
+    }
+
+
 }

Modified: cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/service/RowResolverTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/service/RowResolverTest.java?rev=1133873&r1=1133872&r2=1133873&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/service/RowResolverTest.java
(original)
+++ cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/service/RowResolverTest.java
Thu Jun  9 13:50:27 2011
@@ -23,14 +23,16 @@ package org.apache.cassandra.service;
 
 import java.util.Arrays;
 
-import org.apache.cassandra.SchemaLoader;
 import org.junit.Test;
 
+import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.db.SuperColumn;
 
-import static org.apache.cassandra.db.TableTest.assertColumns;
+import static junit.framework.Assert.*;
 import static org.apache.cassandra.Util.column;
-import static junit.framework.Assert.assertNull;
+import static org.apache.cassandra.Util.superColumn;
+import static org.apache.cassandra.db.TableTest.*;
 
 public class RowResolverTest extends SchemaLoader
 {
@@ -93,4 +95,110 @@ public class RowResolverTest extends Sch
     {
         assertNull(RowRepairResolver.resolveSuperset(Arrays.<ColumnFamily>asList(null,
null)));
     }
+
+    @Test
+    public void testResolveDeleted()
+    {
+        // one CF with columns timestamped before a delete in another cf
+        ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf1.addColumn(column("one", "A", 0));
+
+        ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf2.delete((int) (System.currentTimeMillis() / 1000), 1);
+
+        ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2));
+        // no columns in the cf
+        assertColumns(resolved);
+        assertTrue(resolved.isMarkedForDelete());
+        assertEquals(1, resolved.getMarkedForDeleteAt());
+
+        ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+        scf1.addColumn(superColumn(scf1, "super-foo", column("one", "A", 0)));
+
+        ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+        scf2.delete((int) (System.currentTimeMillis() / 1000), 1);
+
+        ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1,
scf2));
+        // no columns in the cf
+        assertColumns(superResolved);
+        assertTrue(superResolved.isMarkedForDelete());
+        assertEquals(1, superResolved.getMarkedForDeleteAt());
+    }
+
+    @Test
+    public void testResolveDeletedSuper()
+    {
+        // subcolumn is newer than a tombstone on its parent, but not newer than the row
deletion
+        ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+        SuperColumn sc = superColumn(scf1, "super-foo", column("one", "A", 1));
+        sc.markForDeleteAt((int) (System.currentTimeMillis() / 1000), 0);
+        scf1.addColumn(sc);
+
+        ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+        scf2.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+        ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1,
scf2));
+        // no columns in the cf
+        assertColumns(superResolved);
+        assertTrue(superResolved.isMarkedForDelete());
+        assertEquals(2, superResolved.getMarkedForDeleteAt());
+    }
+
+    @Test
+    public void testResolveMultipleDeleted()
+    {
+        // deletes and columns with interleaved timestamp, with out of order return sequence
+
+        ColumnFamily cf1 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf1.delete((int) (System.currentTimeMillis() / 1000), 0);
+
+        // these columns created after the previous deletion
+        ColumnFamily cf2 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf2.addColumn(column("one", "A", 1));
+        cf2.addColumn(column("two", "A", 1));
+
+        //this column created after the next delete
+        ColumnFamily cf3 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf3.addColumn(column("two", "B", 3));
+
+        ColumnFamily cf4 = ColumnFamily.create("Keyspace1", "Standard1");
+        cf4.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+        ColumnFamily resolved = RowRepairResolver.resolveSuperset(Arrays.asList(cf1, cf2,
cf3, cf4));
+        // will have deleted marker and one column
+        assertColumns(resolved, "two");
+        assertColumn(resolved, "two", "B", 3);
+        assertTrue(resolved.isMarkedForDelete());
+        assertEquals(2, resolved.getMarkedForDeleteAt());
+
+
+        ColumnFamily scf1 = ColumnFamily.create("Keyspace1", "Super1");
+        scf1.delete((int) (System.currentTimeMillis() / 1000), 0);
+
+        // these columns created after the previous deletion
+        ColumnFamily scf2 = ColumnFamily.create("Keyspace1", "Super1");
+        scf2.addColumn(superColumn(scf2, "super1", column("one", "A", 1), column("two", "A",
1)));
+
+        //these columns created after the next delete
+        ColumnFamily scf3 = ColumnFamily.create("Keyspace1", "Super1");
+        scf3.addColumn(superColumn(scf3, "super1", column("two", "B", 3)));
+        scf3.addColumn(superColumn(scf3, "super2", column("three", "A", 3), column("four",
"A", 3)));
+
+        ColumnFamily scf4 = ColumnFamily.create("Keyspace1", "Super1");
+        scf4.delete((int) (System.currentTimeMillis() / 1000), 2);
+
+        ColumnFamily superResolved = RowRepairResolver.resolveSuperset(Arrays.asList(scf1,
scf2, scf3, scf4));
+        // will have deleted marker and two super cols
+        assertColumns(superResolved, "super1", "super2");
+
+        assertSubColumns(superResolved, "super1", "two");
+        assertSubColumn(superResolved, "super1", "two", "B", 3);
+
+        assertSubColumns(superResolved, "super2", "four", "three");
+        assertSubColumn(superResolved, "super2", "three", "A", 3);
+        assertSubColumn(superResolved, "super2", "four", "A", 3);
+
+        assertTrue(superResolved.isMarkedForDelete());
+        assertEquals(2, superResolved.getMarkedForDeleteAt());
+    }
 }



Mime
View raw message