cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r987728 - in /cassandra/trunk: CHANGES.txt src/java/org/apache/cassandra/db/ColumnFamilyStore.java test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
Date Sat, 21 Aug 2010 11:37:05 GMT
Author: jbellis
Date: Sat Aug 21 11:37:04 2010
New Revision: 987728

URL: http://svn.apache.org/viewvc?rev=987728&view=rev
Log:
allow index expressions against columns that are not part of the SlicePredicate.
patch by jbellis; reviewed by Nate McCall for CASSANDRA-1410

Modified:
    cassandra/trunk/CHANGES.txt
    cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java

Modified: cassandra/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/trunk/CHANGES.txt?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
--- cassandra/trunk/CHANGES.txt (original)
+++ cassandra/trunk/CHANGES.txt Sat Aug 21 11:37:04 2010
@@ -24,6 +24,8 @@ dev
  * use JNA, if present, to take snapshots (CASSANDRA-1371)
  * truncate hints if starting 0.7 for the first time (CASSANDRA-1414)
  * fix FD leak in single-row slicepredicate queries (CASSANDRA-1416)
+ * allow index expressions against columns that are not part of the 
+   SlicePredicate (CASSANDRA-1410)
 
 
 0.7-beta1

Modified: cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
--- cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java (original)
+++ cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java Sat Aug 21 11:37:04
2010
@@ -897,7 +897,12 @@ public class ColumnFamilyStore implement
      */
     public ColumnFamily getColumnFamily(QueryFilter filter)
     {
-        return getColumnFamily(filter, (int) (System.currentTimeMillis() / 1000) - metadata.gcGraceSeconds);
+        return getColumnFamily(filter, gcBefore());
+    }
+
+    private int gcBefore()
+    {
+        return (int) (System.currentTimeMillis() / 1000) - metadata.gcGraceSeconds;
     }
 
     private ColumnFamily cacheRow(DecoratedKey key)
@@ -1136,15 +1141,68 @@ public class ColumnFamilyStore implement
 
     public List<Row> scan(IndexClause clause, AbstractBounds range, IFilter dataFilter)
     {
+        // Start with the most-restrictive indexed clause, then apply remaining clauses
+        // to each row matching that clause.
         // TODO: allow merge join instead of just one index + loop
-        IndexExpression first = highestSelectivityPredicate(clause);
-        ColumnFamilyStore indexCFS = getIndexedColumnFamilyStore(first.column_name);
+        IndexExpression primary = highestSelectivityPredicate(clause);
+        ColumnFamilyStore indexCFS = getIndexedColumnFamilyStore(primary.column_name);
         assert indexCFS != null;
-        DecoratedKey indexKey = indexCFS.partitioner.decorateKey(first.value);
+        DecoratedKey indexKey = indexCFS.partitioner.decorateKey(primary.value);
+
+        // if the slicepredicate doesn't contain all the columns for which we have expressions
to evaluate,
+        // it needs to be expanded to include those too
+        IFilter firstFilter = dataFilter;
+        NamesQueryFilter extraFilter = null;
+        if (clause.expressions.size() > 1)
+        {
+            if (dataFilter instanceof SliceQueryFilter)
+            {
+                // if we have a high chance of getting all the columns in a single index
slice, do that.
+                // otherwise, create an extraFilter to fetch by name the columns referenced
by the additional expressions.
+                if (getMaxRowSize() < DatabaseDescriptor.getColumnIndexSize())
+                {
+                    firstFilter = new SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY,
+                                                       ArrayUtils.EMPTY_BYTE_ARRAY,
+                                                       ((SliceQueryFilter) dataFilter).reversed,
+                                                       Integer.MAX_VALUE);
+                }
+                else
+                {
+                    SortedSet<byte[]> columns = new TreeSet<byte[]>(getComparator());
+                    for (IndexExpression expr : clause.expressions)
+                    {
+                        if (expr == primary)
+                            continue;
+                        columns.add(expr.column_name);
+                    }
+                    extraFilter = new NamesQueryFilter(columns);
+                }
+            }
+            else
+            {
+                // just add in columns that are not part of the resultset
+                assert dataFilter instanceof NamesQueryFilter;
+                SortedSet<byte[]> columns = new TreeSet<byte[]>(getComparator());
+                for (IndexExpression expr : clause.expressions)
+                {
+                    if (expr == primary || ((NamesQueryFilter) dataFilter).columns.contains(expr.column_name))
+                        continue;
+                    columns.add(expr.column_name);
+                }
+                if (columns.size() > 0)
+                {
+                    columns.addAll(((NamesQueryFilter) dataFilter).columns);
+                    firstFilter = new NamesQueryFilter(columns);
+                }
+            }
+        }
 
         List<Row> rows = new ArrayList<Row>();
         byte[] startKey = clause.start_key;
-        
+        QueryPath path = new QueryPath(columnFamily);
+
+        // fetch row keys matching the primary expression, fetch the slice predicate for
each
+        // and filter by remaining expressions.  repeat until finished w/ assigned range
or index row is exhausted.
         outer:
         while (true)
         {
@@ -1176,9 +1234,38 @@ public class ColumnFamilyStore implement
                     break outer;
                 if (!range.contains(dk.token))
                     continue;
-                ColumnFamily data = getColumnFamily(new QueryFilter(dk, new QueryPath(columnFamily),
dataFilter));
-                if (satisfies(data, clause, first))
+
+                // get the row columns requested, and additional columns for the expressions
if necessary
+                ColumnFamily data = getColumnFamily(new QueryFilter(dk, path, firstFilter));
+                if (extraFilter != null)
+                {
+                    // we might have gotten the expression columns in with the main data
slice, but
+                    // we can't know for sure until that slice is done.  So, we'll do the
extra query
+                    // if we go through and any expression columns are not present.
+                    for (IndexExpression expr : clause.expressions)
+                    {
+                        if (expr != primary && data.getColumn(expr.column_name) ==
null)
+                        {
+                            data.addAll(getColumnFamily(new QueryFilter(dk, path, extraFilter)));
+                            break;
+                        }
+                    }
+                }
+
+                if (satisfies(data, clause, primary))
+                {
+                    // cut the resultset back to what was requested, if necessary
+                    if (firstFilter != dataFilter)
+                    {
+                        ColumnFamily expandedData = data;
+                        data = expandedData.cloneMeShallow();
+                        IColumnIterator iter = dataFilter.getMemtableColumnIterator(expandedData,
dk, getComparator());
+                        new QueryFilter(dk, path, dataFilter).collectCollatedColumns(data,
iter, gcBefore());
+                    }
+
                     rows.add(new Row(dk, data));
+                }
+
                 if (rows.size() == clause.count)
                     break outer;
             }

Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java?rev=987728&r1=987727&r2=987728&view=diff
==============================================================================
--- cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java (original)
+++ cassandra/trunk/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java Sat Aug 21
11:37:04 2010
@@ -179,6 +179,7 @@ public class ColumnFamilyStoreTest exten
         rm.add(new QueryPath("Indexed1", null, "birthdate".getBytes("UTF8")), FBUtilities.toByteArray(3L),
new TimestampClock(0));
         rm.apply();
 
+        // basic single-expression query
         IndexExpression expr = new IndexExpression("birthdate".getBytes("UTF8"), IndexOperator.EQ,
FBUtilities.toByteArray(1L));
         IndexClause clause = new IndexClause(Arrays.asList(expr), ArrayUtils.EMPTY_BYTE_ARRAY,
100);
         IFilter filter = new IdentityQueryFilter();
@@ -193,12 +194,28 @@ public class ColumnFamilyStoreTest exten
         assert Arrays.equals(FBUtilities.toByteArray(1L), rows.get(0).cf.getColumn("birthdate".getBytes("UTF8")).value());
         assert Arrays.equals(FBUtilities.toByteArray(1L), rows.get(1).cf.getColumn("birthdate".getBytes("UTF8")).value());
 
+        // add a second expression
         IndexExpression expr2 = new IndexExpression("notbirthdate".getBytes("UTF8"), IndexOperator.GTE,
FBUtilities.toByteArray(2L));
         clause = new IndexClause(Arrays.asList(expr, expr2), ArrayUtils.EMPTY_BYTE_ARRAY,
100);
         rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
filter);
 
         assert rows.size() == 1 : StringUtils.join(rows, ",");
         assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+
+        // same query again, but with resultset not including the subordinate expression
+        rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
new NamesQueryFilter("birthdate".getBytes("UTF8")));
+
+        assert rows.size() == 1 : StringUtils.join(rows, ",");
+        assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+        assert rows.get(0).cf.getColumnCount() == 1 : rows.get(0).cf;
+
+        // once more, this time with a slice rowset that needs to be expanded
+        SliceQueryFilter sqf = new SliceQueryFilter(ArrayUtils.EMPTY_BYTE_ARRAY, ArrayUtils.EMPTY_BYTE_ARRAY,
false, 0);
+        rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range,
sqf);
+
+        assert rows.size() == 1 : StringUtils.join(rows, ",");
+        assert Arrays.equals("k3".getBytes(), rows.get(0).key.key);
+        assert rows.get(0).cf.getColumnCount() == 0;
     }
 
     @Test



Mime
View raw message