cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject git commit: Fix skipping columns with multiple slices
Date Thu, 03 Oct 2013 09:07:12 GMT
Updated Branches:
  refs/heads/cassandra-1.2 c8f0e3a9f -> 20a805023


Fix skipping columns with multiple slices

patch by frousseau & slebresne; reviewed by frousseau & slebresne for CASSANDRA-6119


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/20a80502
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/20a80502
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/20a80502

Branch: refs/heads/cassandra-1.2
Commit: 20a805023fa26ab1b2f70b574b35357df9652cd3
Parents: c8f0e3a
Author: Sylvain Lebresne <sylvain@datastax.com>
Authored: Thu Oct 3 11:06:12 2013 +0200
Committer: Sylvain Lebresne <sylvain@datastax.com>
Committed: Thu Oct 3 11:06:12 2013 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../db/columniterator/IndexedSliceReader.java   |  17 +-
 .../cassandra/db/ColumnFamilyStoreTest.java     | 198 +++++++++++++++++++
 3 files changed, 213 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cc04eca..c1d1991 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -12,6 +12,7 @@
  * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
  * Fix fat client schema pull NPE (CASSANDRA-6089)
  * Fix memtable flushing for indexed tables (CASSANDRA-6112)
+ * Fix skipping columns with multiple slices (CASSANDRA-6119)
 
 
 1.2.10

http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
index 472ecfc..df916b2 100644
--- a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
+++ b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java
@@ -391,7 +391,7 @@ class IndexedSliceReader extends AbstractIterator<OnDiskAtom> implements
OnDiskA
 
             // scan from index start
             OnDiskAtom column = null;
-            while (file.bytesPastMark(mark) < currentIndex.width)
+            while (file.bytesPastMark(mark) < currentIndex.width || column != null)
             {
                 // Only fetch a new column if we haven't dealt with the previous one.
                 if (column == null)
@@ -467,20 +467,31 @@ class IndexedSliceReader extends AbstractIterator<OnDiskAtom>
implements OnDiskA
             OnDiskAtom.Serializer atomSerializer = emptyColumnFamily.getOnDiskSerializer();
             int columns = file.readInt();
 
-            for (int i = 0; i < columns; i++)
+            OnDiskAtom column = null;
+            int i = 0;
+            while (i < columns || column != null)
             {
-                OnDiskAtom column = atomSerializer.deserializeFromSSTable(file, sstable.descriptor.version);
+                // Only fetch a new column if we haven't dealt with the previous one.
+                if (column == null)
+                {
+                    column = atomSerializer.deserializeFromSSTable(file, sstable.descriptor.version);
+                    i++;
+                }
 
                 // col is before slice
                 // (If in slice, don't bother checking that until we change slice)
                 if (!inSlice && isColumnBeforeSliceStart(column))
+                {
+                    column = null;
                     continue;
+                }
 
                 // col is within slice
                 if (isColumnBeforeSliceFinish(column))
                 {
                     inSlice = true;
                     addColumn(column);
+                    column = null;
                 }
                 // col is after slice. more slices?
                 else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/20a80502/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index abe3f05..cd30297 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -1160,6 +1160,204 @@ public class ColumnFamilyStoreTest extends SchemaLoader
 
     @SuppressWarnings("unchecked")
     @Test
+    public void testMultiRangeSomeEmptyNoIndex() throws Throwable
+    {
+        // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+        // directly instead of using QueryFilter to build it for us
+        ColumnSlice[] ranges = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+                new ColumnSlice(bytes("colC"), bytes("colE")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        ColumnSlice[] rangesReversed = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colE"), bytes("colC")),
+                new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        String tableName = "Keyspace1";
+        String cfName = "Standard1";
+        Table table = Table.open(tableName);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+        cfs.clearUnsafe();
+
+        String[] letters = new String[] { "a", "b", "c", "d", "i" };
+        Column[] cols = new Column[letters.length];
+        for (int i = 0; i < cols.length; i++)
+        {
+            cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+                    ByteBuffer.wrap(new byte[1]), 1);
+        }
+
+        putColsStandard(cfs, dk("a"), cols);
+
+        cfs.forceBlockingFlush();
+
+        SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+        SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false,
3);
+        SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+        SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed,
true, 3);
+
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD",
"colI");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA",
"colC", "colD");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC",
"colA");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI",
"colD", "colC");
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testMultiRangeSomeEmptyIndexed() throws Throwable
+    {
+        // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+        // directly instead of using QueryFilter to build it for us
+        ColumnSlice[] ranges = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+                new ColumnSlice(bytes("colC"), bytes("colE")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        ColumnSlice[] rangesReversed = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colE"), bytes("colC")),
+                new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        String tableName = "Keyspace1";
+        String cfName = "Standard1";
+        Table table = Table.open(tableName);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+        cfs.clearUnsafe();
+
+        String[] letters = new String[] { "a", "b", "c", "d", "i" };
+        Column[] cols = new Column[letters.length];
+        for (int i = 0; i < cols.length; i++)
+        {
+            cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+                    ByteBuffer.wrap(new byte[1366]), 1);
+        }
+
+        putColsStandard(cfs, dk("a"), cols);
+
+        cfs.forceBlockingFlush();
+
+        SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+        SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false,
3);
+        SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+        SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed,
true, 3);
+
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD",
"colI");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA",
"colC", "colD");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colD", "colC",
"colA");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI",
"colD", "colC");
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testMultiRangeContiguousNoIndex() throws Throwable
+    {
+        // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+        // directly instead of using QueryFilter to build it for us
+        ColumnSlice[] ranges = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+                new ColumnSlice(bytes("colC"), bytes("colE")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        ColumnSlice[] rangesReversed = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colE"), bytes("colC")),
+                new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        String tableName = "Keyspace1";
+        String cfName = "Standard1";
+        Table table = Table.open(tableName);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+        cfs.clearUnsafe();
+
+        String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" };
+        Column[] cols = new Column[letters.length];
+        for (int i = 0; i < cols.length; i++)
+        {
+            cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+                    ByteBuffer.wrap(new byte[1]), 1);
+        }
+
+        putColsStandard(cfs, dk("a"), cols);
+
+        cfs.forceBlockingFlush();
+
+        SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+        SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false,
3);
+        SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+        SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed,
true, 3);
+
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD",
"colE", "colF", "colG", "colI");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA",
"colC", "colD");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF",
"colE", "colD", "colC", "colA");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI",
"colG", "colF");
+
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testMultiRangeContiguousIndexed() throws Throwable
+    {
+        // in order not to change thrift interfaces at this stage we build SliceQueryFilter
+        // directly instead of using QueryFilter to build it for us
+        ColumnSlice[] ranges = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colA")),
+                new ColumnSlice(bytes("colC"), bytes("colE")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colI"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        ColumnSlice[] rangesReversed = new ColumnSlice[] {
+                new ColumnSlice(ByteBuffer.wrap(EMPTY_BYTE_ARRAY), bytes("colI")),
+                new ColumnSlice(bytes("colG"), bytes("colG")),
+                new ColumnSlice(bytes("colF"), bytes("colF")),
+                new ColumnSlice(bytes("colE"), bytes("colC")),
+                new ColumnSlice(bytes("colA"), ByteBuffer.wrap(EMPTY_BYTE_ARRAY)) };
+
+        String tableName = "Keyspace1";
+        String cfName = "Standard1";
+        Table table = Table.open(tableName);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
+        cfs.clearUnsafe();
+
+        String[] letters = new String[] { "a", "b", "c", "d", "e", "f", "g", "h", "i" };
+        Column[] cols = new Column[letters.length];
+        for (int i = 0; i < cols.length; i++)
+        {
+            cols[i] = new Column(ByteBufferUtil.bytes("col" + letters[i].toUpperCase()),
+                    ByteBuffer.wrap(new byte[1366]), 1);
+        }
+
+        putColsStandard(cfs, dk("a"), cols);
+
+        cfs.forceBlockingFlush();
+
+        SliceQueryFilter multiRangeForward = new SliceQueryFilter(ranges, false, 100);
+        SliceQueryFilter multiRangeForwardWithCounting = new SliceQueryFilter(ranges, false,
3);
+        SliceQueryFilter multiRangeReverse = new SliceQueryFilter(rangesReversed, true, 100);
+        SliceQueryFilter multiRangeReverseWithCounting = new SliceQueryFilter(rangesReversed,
true, 3);
+
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForward, "a", "colA", "colC", "colD",
"colE", "colF", "colG", "colI");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeForwardWithCounting, "a", "colA",
"colC", "colD");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverse, "a", "colI", "colG", "colF",
"colE", "colD", "colC", "colA");
+        findRowGetSlicesAndAssertColsFound(cfs, multiRangeReverseWithCounting, "a", "colI",
"colG", "colF");
+
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
     public void testMultiRangeIndexed() throws Throwable
     {
         // in order not to change thrift interfaces at this stage we build SliceQueryFilter


Mime
View raw message