cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject [4/6] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Date Fri, 29 Apr 2016 11:01:28 GMT
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/943e732c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/943e732c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/943e732c

Branch: refs/heads/trunk
Commit: 943e732caeb1da59e5e9af5479cd9d75ca9d9030
Parents: 474d3bf dff7b44
Author: Sam Tunnicliffe <sam@beobal.com>
Authored: Fri Apr 29 11:50:03 2016 +0100
Committer: Sam Tunnicliffe <sam@beobal.com>
Committed: Fri Apr 29 11:55:07 2016 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../restrictions/PrimaryKeyRestrictionSet.java  | 71 +++++++++++++++++++-
 .../restrictions/StatementRestrictions.java     | 34 +++++++---
 .../SelectSingleColumnRelationTest.java         |  4 ++
 4 files changed, 97 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 268d011,fb06cd6..3184cce
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,19 -1,5 +1,20 @@@
 -2.2.7
 +3.0.6
 + * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669)
 + * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600)
 + * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654)
 + * Ignore all LocalStrategy keyspaces for streaming and other related
 +   operations (CASSANDRA-11627)
 + * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523)
 + * Only open one sstable scanner per sstable (CASSANDRA-11412)
 + * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410)
 + * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485)
 + * LogAwareFileLister should only use OLD sstable files in current folder to determine disk
consistency (CASSANDRA-11470)
 + * Notify indexers of expired rows during compaction (CASSANDRA-11329)
 + * Properly respond with ProtocolError when a v1/v2 native protocol
 +   header is received (CASSANDRA-11464)
 + * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120)
 +Merged from 2.2:
+  * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510)
   * JSON datetime formatting needs timezone (CASSANDRA-11137)
   * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502)
   * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java
index 8121858,0c10f13..e063f91
--- a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java
@@@ -64,27 -67,48 +64,58 @@@ final class PrimaryKeyRestrictionSet ex
       */
      private boolean contains;
  
 +    /**
 +     * <code>true</code> if the restrictions corresponding to a partition key,
<code>false</code> if it's clustering columns.
 +     */
 +    private boolean isPartitionKey;
  
+     /**
+      * If restrictions apply to clustering columns, we need to check whether they can be
satisfied by an index lookup
+      * as this affects which other restrictions can legally be specified (if an index is
present, we are more lenient
+      * about what additional filtering can be performed on the results of a lookup - see
CASSANDRA-11510).
+      *
+      * We don't hold a reference to the SecondaryIndexManager itself as this is not strictly
a singleton (although
+      * we often treat is as one), the field would also require annotation with @Unmetered
to avoid blowing up the
+      * object size (used when calculating the size of prepared statements for caching).
Instead, we refer to the
+      * CFMetaData and retrieve the index manager when necessary.
+      *
+      * There are a couple of scenarios where the CFM can be null (and we make sure and test
for null when we use it):
+      *  * where an empty set of restrictions are created for use in processing query results
- see
+      *    SelectStatement.forSelection
+      *  * where the restrictions apply to partition keys and not clustering columns e.g.
+      *    StatementRestrictions.partitionKeyRestrictions
+      *  * in unit tests (in particular PrimaryKeyRestrictionSetTest which is primarily concerned
with the correct
+      *    generation of bounds when secondary indexes are not used).
+      */
+     private final CFMetaData cfm;
+ 
 -    public PrimaryKeyRestrictionSet(CType ctype)
 +    public PrimaryKeyRestrictionSet(ClusteringComparator comparator, boolean isPartitionKey)
      {
 -        this(ctype, null);
++       this(comparator, isPartitionKey, null);
+     }
+ 
 -    public PrimaryKeyRestrictionSet(CType ctype, CFMetaData cfm)
++    public PrimaryKeyRestrictionSet(ClusteringComparator comparator, boolean isPartitionKey,
CFMetaData cfm)
+     {
 -        super(ctype);
 +        super(comparator);
++
++        if (cfm != null)
++            assert !isPartitionKey;
++
+         this.cfm = cfm;
          this.restrictions = new RestrictionSet();
          this.eq = true;
 +        this.isPartitionKey = isPartitionKey;
      }
  
      private PrimaryKeyRestrictionSet(PrimaryKeyRestrictionSet primaryKeyRestrictions,
-                                                Restriction restriction) throws InvalidRequestException
+                                      Restriction restriction) throws InvalidRequestException
      {
 -        super(primaryKeyRestrictions.ctype);
 +        super(primaryKeyRestrictions.comparator);
          this.restrictions = primaryKeyRestrictions.restrictions.addRestriction(restriction);
 +        this.isPartitionKey = primaryKeyRestrictions.isPartitionKey;
+         this.cfm = primaryKeyRestrictions.cfm;
  
-         if (!primaryKeyRestrictions.isEmpty())
+         if (!primaryKeyRestrictions.isEmpty() && !hasSupportingIndex(restriction))
          {
              ColumnDefinition lastRestrictionStart = primaryKeyRestrictions.restrictions.lastRestriction().getFirstColumn();
              ColumnDefinition newRestrictionStart = restriction.getFirstColumn();
@@@ -110,15 -134,12 +141,22 @@@
              this.eq = true;
      }
  
 +    private List<ByteBuffer> toByteBuffers(SortedSet<? extends ClusteringPrefix>
clusterings)
 +    {
 +        // It's currently a tad hard to follow that this is only called for partition key
so we should fix that
 +        List<ByteBuffer> l = new ArrayList<>(clusterings.size());
 +        for (ClusteringPrefix clustering : clusterings)
 +            l.add(CFMetaData.serializePartitionKey(clustering));
 +        return l;
 +    }
 +
+     private boolean hasSupportingIndex(Restriction restriction)
+     {
+         return cfm != null
 -               && restriction.hasSupportingIndex(Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager);
++               && restriction.hasSupportingIndex(Keyspace.openAndGetStore(cfm).indexManager);
++
+     }
+ 
      @Override
      public boolean isSlice()
      {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 763a7be,5b7c58d..942d1f9
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@@ -125,53 -112,28 +126,58 @@@ public final class StatementRestriction
                                   VariableSpecifications boundNames,
                                   boolean selectsOnlyStaticColumns,
                                   boolean selectACollection,
 -                                 boolean useFiltering)
 +                                 boolean useFiltering,
 +                                 boolean forView) throws InvalidRequestException
      {
-         this(type, cfm);
++        this.type = type;
+         this.cfm = cfm;
 -        this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsCType());
 -        this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator,
cfm);
++        this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsClusteringComparator(),
true);
++        this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator,
false, cfm);
+         this.nonPrimaryKeyRestrictions = new RestrictionSet();
++        this.notNullColumns = new HashSet<>();
  
          /*
 -         * WHERE clause. For a given entity, rules are: - EQ relation conflicts with anything
else (including a 2nd EQ)
 -         * - Can't have more than one LT(E) relation (resp. GT(E) relation) - IN relation
are restricted to row keys
 -         * (for now) and conflicts with anything else (we could allow two IN for the same
entity but that doesn't seem
 -         * very useful) - The value_alias cannot be restricted in any way (we don't support
wide rows with indexed value
 -         * in CQL so far)
 +         * WHERE clause. For a given entity, rules are:
 +         *   - EQ relation conflicts with anything else (including a 2nd EQ)
 +         *   - Can't have more than one LT(E) relation (resp. GT(E) relation)
 +         *   - IN relation are restricted to row keys (for now) and conflicts with anything
else (we could
 +         *     allow two IN for the same entity but that doesn't seem very useful)
 +         *   - The value_alias cannot be restricted in any way (we don't support wide rows
with indexed value
 +         *     in CQL so far)
           */
 -        for (Relation relation : whereClause)
 -            addRestriction(relation.toRestriction(cfm, boundNames));
 +        for (Relation relation : whereClause.relations)
 +        {
 +            if (relation.operator() == Operator.IS_NOT)
 +            {
 +                if (!forView)
 +                    throw new InvalidRequestException("Unsupported restriction: " + relation);
 +
 +                for (ColumnDefinition def : relation.toRestriction(cfm, boundNames).getColumnDefs())
 +                    this.notNullColumns.add(def);
 +            }
 +            else
 +            {
 +                addRestriction(relation.toRestriction(cfm, boundNames));
 +            }
 +        }
 +
 +        boolean hasQueriableClusteringColumnIndex = false;
 +        boolean hasQueriableIndex = false;
  
 -        SecondaryIndexManager secondaryIndexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName).indexManager;
 -        boolean hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(secondaryIndexManager);
 -        boolean hasQueriableIndex = hasQueriableClusteringColumnIndex
 -                || partitionKeyRestrictions.hasSupportingIndex(secondaryIndexManager)
 -                || nonPrimaryKeyRestrictions.hasSupportingIndex(secondaryIndexManager);
 +        if (type.allowUseOfSecondaryIndices())
 +        {
 +            ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
 +            SecondaryIndexManager secondaryIndexManager = cfs.indexManager;
 +
 +            if (whereClause.containsCustomExpressions())
 +                processCustomIndexExpressions(whereClause.expressions, boundNames, secondaryIndexManager);
 +
 +            hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(secondaryIndexManager);
 +            hasQueriableIndex = !indexRestrictions.getCustomIndexExpressions().isEmpty()
 +                    || hasQueriableClusteringColumnIndex
 +                    || partitionKeyRestrictions.hasSupportingIndex(secondaryIndexManager)
 +                    || nonPrimaryKeyRestrictions.hasSupportingIndex(secondaryIndexManager);
 +        }
  
          // At this point, the select statement if fully constructed, but we still have a
few things to validate
          processPartitionKeyRestrictions(hasQueriableIndex);
@@@ -430,130 -301,61 +436,136 @@@
       * Processes the clustering column restrictions.
       *
       * @param hasQueriableIndex <code>true</code> if some of the queried data
are indexed, <code>false</code> otherwise
 +     * @param selectsOnlyStaticColumns <code>true</code> if the selected or
modified columns are all statics,
 +     * <code>false</code> otherwise.
       * @param selectACollection <code>true</code> if the query should return
a collection column
 -     * @throws InvalidRequestException if the request is invalid
       */
      private void processClusteringColumnsRestrictions(boolean hasQueriableIndex,
 -                                                      boolean selectACollection) throws
InvalidRequestException
 +                                                      boolean selectsOnlyStaticColumns,
 +                                                      boolean selectACollection,
 +                                                      boolean forView) throws InvalidRequestException
      {
 -        checkFalse(clusteringColumnsRestrictions.isIN() && selectACollection,
 -                   "Cannot restrict clustering columns by IN relations when a collection
is selected by the query");
 -        checkFalse(clusteringColumnsRestrictions.isContains() && !hasQueriableIndex,
 -                   "Cannot restrict clustering columns by a CONTAINS relation without a
secondary index");
 +        checkFalse(!type.allowClusteringColumnSlices() && clusteringColumnsRestrictions.isSlice(),
 +                   "Slice restrictions are not supported on the clustering columns in %s
statements", type);
  
 -        if (hasClusteringColumnsRestriction() && clusteringRestrictionsNeedFiltering())
 +        if (!type.allowClusteringColumnSlices()
 +               && (!cfm.isCompactTable() || (cfm.isCompactTable() && !hasClusteringColumnsRestriction())))
          {
 -            if (hasQueriableIndex)
 -            {
 -                usesSecondaryIndexing = true;
 -                return;
 -            }
 -
 -            List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns();
 -            List<ColumnDefinition> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs());
 +            if (!selectsOnlyStaticColumns && hasUnrestrictedClusteringColumns())
 +                throw invalidRequest("Some clustering keys are missing: %s",
 +                                     Joiner.on(", ").join(getUnrestrictedClusteringColumns()));
 +        }
 +        else
 +        {
 +            checkFalse(clusteringColumnsRestrictions.isIN() && selectACollection,
 +                       "Cannot restrict clustering columns by IN relations when a collection
is selected by the query");
 +            checkFalse(clusteringColumnsRestrictions.isContains() && !hasQueriableIndex,
 +                       "Cannot restrict clustering columns by a CONTAINS relation without
a secondary index");
  
-             if (hasClusteringColumnsRestriction())
 -            for (int i = 0, m = restrictedColumns.size(); i < m; i++)
++            if (hasClusteringColumnsRestriction() && clusteringRestrictionsNeedFiltering())
              {
 -                ColumnDefinition clusteringColumn = clusteringColumns.get(i);
 -                ColumnDefinition restrictedColumn = restrictedColumns.get(i);
++                if (hasQueriableIndex || forView)
++                {
++                    usesSecondaryIndexing = true;
++                    return;
++                }
++
 +                List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns();
 +                List<ColumnDefinition> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs());
  
 -                if (!clusteringColumn.equals(restrictedColumn))
 +                for (int i = 0, m = restrictedColumns.size(); i < m; i++)
                  {
 -                    throw invalidRequest(
 -                              "PRIMARY KEY column \"%s\" cannot be restricted as preceding
column \"%s\" is not restricted",
 -                              restrictedColumn.name,
 -                              clusteringColumn.name);
 +                    ColumnDefinition clusteringColumn = clusteringColumns.get(i);
 +                    ColumnDefinition restrictedColumn = restrictedColumns.get(i);
 +
 +                    if (!clusteringColumn.equals(restrictedColumn))
 +                    {
-                         checkTrue(hasQueriableIndex || forView,
-                                   "PRIMARY KEY column \"%s\" cannot be restricted as preceding
column \"%s\" is not restricted",
-                                   restrictedColumn.name,
-                                   clusteringColumn.name);
- 
-                         usesSecondaryIndexing = true; // handle gaps and non-keyrange cases.
-                         break;
++                        throw invalidRequest(
++                           "PRIMARY KEY column \"%s\" cannot be restricted as preceding
column \"%s\" is not restricted",
++                            restrictedColumn.name,
++                            clusteringColumn.name);
 +                    }
                  }
              }
          }
+     }
  
-         if (clusteringColumnsRestrictions.isContains())
-             usesSecondaryIndexing = true;
+     public final boolean clusteringRestrictionsNeedFiltering()
+     {
+         assert clusteringColumnsRestrictions instanceof PrimaryKeyRestrictionSet;
+         return ((PrimaryKeyRestrictionSet) clusteringColumnsRestrictions).needsFiltering();
      }
  
 -    public List<IndexExpression> getIndexExpressions(SecondaryIndexManager indexManager,
 -                                                     QueryOptions options) throws InvalidRequestException
 +    /**
 +     * Returns the clustering columns that are not restricted.
 +     * @return the clustering columns that are not restricted.
 +     */
 +    private Collection<ColumnIdentifier> getUnrestrictedClusteringColumns()
 +    {
 +        List<ColumnDefinition> missingClusteringColumns = new ArrayList<>(cfm.clusteringColumns());
 +        missingClusteringColumns.removeAll(new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs()));
 +        return ColumnDefinition.toIdentifiers(missingClusteringColumns);
 +    }
 +
 +    /**
 +     * Checks if some clustering columns are not restricted.
 +     * @return <code>true</code> if some clustering columns are not restricted,
<code>false</code> otherwise.
 +     */
 +    private boolean hasUnrestrictedClusteringColumns()
 +    {
 +        return cfm.clusteringColumns().size() != clusteringColumnsRestrictions.size();
 +    }
 +
 +    private void processCustomIndexExpressions(List<CustomIndexExpression> expressions,
 +                                               VariableSpecifications boundNames,
 +                                               SecondaryIndexManager indexManager)
 +    {
 +        if (!MessagingService.instance().areAllNodesAtLeast30())
 +            throw new InvalidRequestException("Please upgrade all nodes to at least 3.0
before using custom index expressions");
 +
 +        if (expressions.size() > 1)
 +            throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS);
 +
 +        CustomIndexExpression expression = expressions.get(0);
 +
 +        CFName cfName = expression.targetIndex.getCfName();
 +        if (cfName.hasKeyspace()
 +            && !expression.targetIndex.getKeyspace().equals(cfm.ksName))
 +            throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm);
 +
 +        if (cfName.getColumnFamily() != null && !cfName.getColumnFamily().equals(cfm.cfName))
 +            throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm);
 +
 +        if (!cfm.getIndexes().has(expression.targetIndex.getIdx()))
 +            throw IndexRestrictions.indexNotFound(expression.targetIndex, cfm);
 +
 +        Index index = indexManager.getIndex(cfm.getIndexes().get(expression.targetIndex.getIdx()).get());
 +
 +        if (!index.getIndexMetadata().isCustom())
 +            throw IndexRestrictions.nonCustomIndexInExpression(expression.targetIndex);
 +
 +        AbstractType<?> expressionType = index.customExpressionValueType();
 +        if (expressionType == null)
 +            throw IndexRestrictions.customExpressionNotSupported(expression.targetIndex);
 +
 +        expression.prepareValue(cfm, expressionType, boundNames);
 +
 +        indexRestrictions.add(expression);
 +    }
 +
 +    public RowFilter getRowFilter(SecondaryIndexManager indexManager, QueryOptions options)
      {
 -        if (!usesSecondaryIndexing || indexRestrictions.isEmpty())
 -            return Collections.emptyList();
 +        if (indexRestrictions.isEmpty())
 +            return RowFilter.NONE;
 +
 +        RowFilter filter = RowFilter.create();
 +        for (Restrictions restrictions : indexRestrictions.getRestrictions())
 +            restrictions.addRowFilterTo(filter, indexManager, options);
  
 -        List<IndexExpression> expressions = new ArrayList<>();
 -        for (Restrictions restrictions : indexRestrictions)
 -            restrictions.addIndexExpressionTo(expressions, indexManager, options);
 +        for (CustomIndexExpression expression : indexRestrictions.getCustomIndexExpressions())
 +            expression.addToRowFilter(filter, cfm, options);
  
 -        return expressions;
 +        return filter;
      }
  
      /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java
----------------------------------------------------------------------


Mime
View raw message