cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ifesdj...@apache.org
Subject [6/6] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Date Wed, 12 Jul 2017 13:53:38 GMT
Merge branch 'cassandra-3.11' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f48a319a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f48a319a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f48a319a

Branch: refs/heads/trunk
Commit: f48a319ac884ef8d6eb54db3176ea2acf627bb89
Parents: 86964da 29db251
Author: Alex Petrov <oleksandr.petrov@gmail.com>
Authored: Wed Jul 12 15:53:10 2017 +0200
Committer: Alex Petrov <oleksandr.petrov@gmail.com>
Committed: Wed Jul 12 15:53:10 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../db/SinglePartitionReadCommand.java          |  30 ++-
 .../db/rows/UnfilteredRowIterators.java         |   3 +-
 .../cassandra/db/transform/BaseIterator.java    |  13 +-
 .../cassandra/db/transform/BasePartitions.java  |   2 +-
 .../apache/cassandra/db/transform/BaseRows.java |   4 +-
 .../apache/cassandra/db/transform/MoreRows.java |   6 +
 .../db/transform/StoppingTransformation.java    |  31 ++--
 .../cassandra/db/transform/Transformation.java  |  24 +++
 .../cassandra/db/transform/UnfilteredRows.java  |   7 +-
 .../apache/cassandra/db/RowCacheCQLTest.java    |  71 +++++++
 .../db/rows/UnfilteredRowIteratorsTest.java     | 185 +++++++++++++++++++
 12 files changed, 356 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 8625d82,30fa350..3d52874
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -98,6 -2,8 +98,7 @@@
   * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
   * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
  Merged from 3.0:
 -3.0.15
+  * Make concat work with iterators that have different subsets of columns (CASSANDRA-13482)
   * Set test.runners based on cores and memory size (CASSANDRA-13078)
   * Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
   * Fix secondary index queries on COMPACT tables (CASSANDRA-13627)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIterators.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/transform/BaseRows.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/transform/MoreRows.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/transform/MoreRows.java
index 786e215,118739b..f3856c4
--- a/src/java/org/apache/cassandra/db/transform/MoreRows.java
+++ b/src/java/org/apache/cassandra/db/transform/MoreRows.java
@@@ -20,6 -20,7 +20,7 @@@
   */
  package org.apache.cassandra.db.transform;
  
 -import org.apache.cassandra.db.PartitionColumns;
++import org.apache.cassandra.db.RegularAndStaticColumns;
  import org.apache.cassandra.db.rows.BaseRowIterator;
  import org.apache.cassandra.db.rows.RowIterator;
  import org.apache.cassandra.db.rows.UnfilteredRowIterator;
@@@ -47,6 -48,11 +48,11 @@@ public interface MoreRows<I extends Bas
          return add(mutable(iterator), more);
      }
  
 -    public static UnfilteredRowIterator extend(UnfilteredRowIterator iterator, MoreRows<?
super UnfilteredRowIterator> more, PartitionColumns columns)
++    public static UnfilteredRowIterator extend(UnfilteredRowIterator iterator, MoreRows<?
super UnfilteredRowIterator> more, RegularAndStaticColumns columns)
+     {
+         return add(Transformation.wrapIterator(iterator, columns), more);
+     }
+ 
      public static RowIterator extend(RowIterator iterator, MoreRows<? super RowIterator>
more)
      {
          return add(mutable(iterator), more);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/transform/Transformation.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/transform/Transformation.java
index 811932c,77f91e4..41c76df
--- a/src/java/org/apache/cassandra/db/transform/Transformation.java
+++ b/src/java/org/apache/cassandra/db/transform/Transformation.java
@@@ -162,6 -162,6 +162,7 @@@ public abstract class Transformation<I 
                 ? (UnfilteredRows) iterator
                 : new UnfilteredRows(iterator);
      }
++
      static FilteredRows mutable(RowIterator iterator)
      {
          return iterator instanceof FilteredRows
@@@ -169,6 -169,29 +170,29 @@@
                 : new FilteredRows(iterator);
      }
  
+     /**
+      * Even though this method is sumilar to `mutable`, it supresses the optimisation of
avoiding creating an additional
+      * wrapping interator object (which both creates an extra object and grows the call
stack during the iteration), it
+      * should be used with caution.
+      *
+      * It is useful in cases when the input has to be checked for more contents rather than
directly checking if it
+      * is stopped. For example, when concatenating two iterators (pseudocode):
+      *
+      *    iter1 = [row(1), row(2), row(3)]
+      *    iter2 = [row(4), row(5), row(6)]
+      *
+      *    UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1), DataLimits.cqlLimits(1).filter(iter1))
+      *
+      * Which should yield two rows: [row(1), row(4)].
+      *
+      * Using stacked transformations instead of wrapping would result into returning a single
row, since the first
+      * iterator will signal the iterator is stopped.
+      */
 -    static UnfilteredRows wrapIterator(UnfilteredRowIterator iterator, PartitionColumns
columns)
++    static UnfilteredRows wrapIterator(UnfilteredRowIterator iterator, RegularAndStaticColumns
columns)
+     {
+         return new UnfilteredRows(iterator, columns);
+     }
+ 
      static <E extends BaseIterator> E add(E to, Transformation add)
      {
          to.add(add);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
index a3dc96e,2dccad7..b8720fc
--- a/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
+++ b/src/java/org/apache/cassandra/db/transform/UnfilteredRows.java
@@@ -33,8 -33,13 +33,13 @@@ final class UnfilteredRows extends Base
  
      public UnfilteredRows(UnfilteredRowIterator input)
      {
+         this(input, input.columns());
+     }
+ 
 -    public UnfilteredRows(UnfilteredRowIterator input, PartitionColumns columns)
++    public UnfilteredRows(UnfilteredRowIterator input, RegularAndStaticColumns columns)
+     {
          super(input);
-         regularAndStaticColumns = input.columns();
 -        partitionColumns = columns;
++        regularAndStaticColumns = columns;
          partitionLevelDeletion = input.partitionLevelDeletion();
      }
  

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
index a3c0e25,a8f7e3d..27cfebd
--- a/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/RowCacheCQLTest.java
@@@ -15,6 -15,6 +15,7 @@@
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
++
  package org.apache.cassandra.db;
  
  import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/f48a319a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
index 0000000,abd2ff7..9e0dd0c
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorsTest.java
@@@ -1,0 -1,189 +1,185 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.cassandra.db.rows;
+ 
+ import java.util.Arrays;
+ import java.util.Iterator;
+ 
+ import org.junit.Test;
+ 
+ import junit.framework.Assert;
+ import org.apache.cassandra.Util;
 -import org.apache.cassandra.config.CFMetaData;
 -import org.apache.cassandra.config.ColumnDefinition;
 -import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.db.BufferDecoratedKey;
+ import org.apache.cassandra.db.DecoratedKey;
+ import org.apache.cassandra.db.DeletionTime;
+ import org.apache.cassandra.db.EmptyIterators;
 -import org.apache.cassandra.db.PartitionColumns;
++import org.apache.cassandra.db.RegularAndStaticColumns;
+ import org.apache.cassandra.db.filter.DataLimits;
+ import org.apache.cassandra.db.marshal.Int32Type;
+ import org.apache.cassandra.dht.Murmur3Partitioner;
++import org.apache.cassandra.schema.ColumnMetadata;
++import org.apache.cassandra.schema.TableMetadata;
+ import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.apache.cassandra.utils.FBUtilities;
+ 
+ public class UnfilteredRowIteratorsTest
+ {
 -    static final CFMetaData metadata;
 -    static final ColumnDefinition v1Metadata;
 -    static final ColumnDefinition v2Metadata;
++    static final TableMetadata metadata;
++    static final ColumnMetadata v1Metadata;
++    static final ColumnMetadata v2Metadata;
+ 
+     static
+     {
 -        // Required because of metadata creation, assertion is thrown otherwise
 -        DatabaseDescriptor.daemonInitialization();
 -
 -        metadata = CFMetaData.Builder.create("", "")
 -                             .addPartitionKey("pk", Int32Type.instance)
 -                                     .addClusteringColumn("ck", Int32Type.instance)
 -                             .addRegularColumn("v1", Int32Type.instance)
 -                             .addRegularColumn("v2", Int32Type.instance)
 -                             .build();
 -        v1Metadata = metadata.partitionColumns().columns(false).getSimple(0);
 -        v2Metadata = metadata.partitionColumns().columns(false).getSimple(1);
++        metadata = TableMetadata.builder("", "")
++                                .addPartitionKeyColumn("pk", Int32Type.instance)
++                                .addClusteringColumn("ck", Int32Type.instance)
++                                .addRegularColumn("v1", Int32Type.instance)
++                                .addRegularColumn("v2", Int32Type.instance)
++                                .build();
++        v1Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(0);
++        v2Metadata = metadata.regularAndStaticColumns().columns(false).getSimple(1);
+     }
+ 
+ 
+     @Test
+     public void concatTest()
+     {
+         UnfilteredRowIterator iter1, iter2, iter3, concat;
+         // simple concatenation
 -        iter1 = rows(metadata.partitionColumns(), 1,
++        iter1 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                      row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
 -        iter2 = rows(metadata.partitionColumns(), 1,
++        iter2 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+                      row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+         concat = UnfilteredRowIterators.concat(iter1, iter2);
 -        Assert.assertEquals(concat.columns(), metadata.partitionColumns());
++        Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
+         assertRows(concat,
+                    row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                    row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)),
+                    row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+                    row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+ 
+         // concat with RHS empty iterator
 -        iter1 = rows(metadata.partitionColumns(), 1,
++        iter1 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                      row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
 -        Assert.assertEquals(concat.columns(), metadata.partitionColumns());
++        Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
+         assertRows(UnfilteredRowIterators.concat(iter1, EmptyIterators.unfilteredRow(metadata,
dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE)),
+                    row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                    row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ 
+         // concat with LHS empty iterator
 -        iter1 = rows(metadata.partitionColumns(), 1,
++        iter1 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                      row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
 -        Assert.assertEquals(concat.columns(), metadata.partitionColumns());
++        Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
+         assertRows(UnfilteredRowIterators.concat(EmptyIterators.unfilteredRow(metadata,
dk(1), false, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE), iter1),
+                    row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                    row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
+ 
+         // concat with different columns
 -        iter1 = rows(metadata.partitionColumns().without(v1Metadata), 1,
++        iter1 = rows(metadata.regularAndStaticColumns().without(v1Metadata), 1,
+                      row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)));
 -        iter2 = rows(metadata.partitionColumns().without(v2Metadata), 1,
++        iter2 = rows(metadata.regularAndStaticColumns().without(v2Metadata), 1,
+                      row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
+         concat = UnfilteredRowIterators.concat(iter1, iter2);
 -        Assert.assertEquals(concat.columns(), PartitionColumns.of(v1Metadata).mergeTo(PartitionColumns.of(v2Metadata)));
++        Assert.assertEquals(concat.columns(), RegularAndStaticColumns.of(v1Metadata).mergeTo(RegularAndStaticColumns.of(v2Metadata)));
+         assertRows(concat,
+                    row(1, cell(v2Metadata, 1)), row(2, cell(v2Metadata, 2)),
+                    row(3, cell(v1Metadata, 3)), row(4, cell(v1Metadata, 4)));
+ 
+         // concat with CQL limits
 -        iter1 = rows(metadata.partitionColumns(), 1,
++        iter1 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                      row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
 -        iter2 = rows(metadata.partitionColumns(), 1,
++        iter2 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+                      row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+         concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds()),
+                                                DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds()));
 -        Assert.assertEquals(concat.columns(), metadata.partitionColumns());
++        Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
+         assertRows(concat,
+                    row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                    row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)));
+ 
+         // concat concatenated iterators
 -        iter1 = rows(metadata.partitionColumns(), 1,
++        iter1 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                      row(2, cell(v1Metadata, 2), cell(v2Metadata, 2)));
 -        iter2 = rows(metadata.partitionColumns(), 1,
++        iter2 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+                      row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+ 
+         concat = UnfilteredRowIterators.concat(DataLimits.cqlLimits(1).filter(iter1, FBUtilities.nowInSeconds()),
+                                                DataLimits.cqlLimits(1).filter(iter2, FBUtilities.nowInSeconds()));
+ 
 -        iter3 = rows(metadata.partitionColumns(), 1,
++        iter3 = rows(metadata.regularAndStaticColumns(), 1,
+                      row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)),
+                      row(5, cell(v1Metadata, 5), cell(v2Metadata, 5)));
+         concat = UnfilteredRowIterators.concat(concat, DataLimits.cqlLimits(1).filter(iter3,
FBUtilities.nowInSeconds()));
+ 
 -        Assert.assertEquals(concat.columns(), metadata.partitionColumns());
++        Assert.assertEquals(concat.columns(), metadata.regularAndStaticColumns());
+         assertRows(concat,
+                    row(1, cell(v1Metadata, 1), cell(v2Metadata, 1)),
+                    row(3, cell(v1Metadata, 3), cell(v2Metadata, 3)),
+                    row(4, cell(v1Metadata, 4), cell(v2Metadata, 4)));
+     }
+ 
+     public static void assertRows(UnfilteredRowIterator iterator, Row... rows)
+     {
+         Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
+ 
+         while (iterator.hasNext() && rowsIterator.hasNext())
+             Assert.assertEquals(iterator.next(), rowsIterator.next());
+ 
+         Assert.assertTrue(iterator.hasNext() == rowsIterator.hasNext());
+     }
+ 
+     public static DecoratedKey dk(int pk)
+     {
+         return new BufferDecoratedKey(new Murmur3Partitioner.LongToken(pk), ByteBufferUtil.bytes(pk));
+     }
+ 
 -    public static UnfilteredRowIterator rows(PartitionColumns columns, int pk, Row... rows)
++    public static UnfilteredRowIterator rows(RegularAndStaticColumns columns, int pk, Row...
rows)
+     {
+         Iterator<Row> rowsIterator = Arrays.asList(rows).iterator();
+         return new AbstractUnfilteredRowIterator(metadata, dk(pk), DeletionTime.LIVE, columns,
Rows.EMPTY_STATIC_ROW, false, EncodingStats.NO_STATS) {
+             protected Unfiltered computeNext()
+             {
+                 return rowsIterator.hasNext() ? rowsIterator.next() : endOfData();
+             }
+         };
+     }
+ 
+     public Row row(int ck, Cell... columns)
+     {
+         BTreeRow.Builder builder = new BTreeRow.Builder(true);
+         builder.newRow(Util.clustering(metadata.comparator, ck));
+         for (Cell cell : columns)
+             builder.addCell(cell);
+         return builder.build();
+     }
+ 
 -    public Cell cell(ColumnDefinition metadata, int v)
++    public Cell cell(ColumnMetadata metadata, int v)
+     {
+         return new BufferCell(metadata,
+                               1L, BufferCell.NO_TTL, BufferCell.NO_DELETION_TIME, ByteBufferUtil.bytes(v),
null);
+     }
+ }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message