cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [08/10] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Date Tue, 26 Apr 2016 07:34:28 GMT
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bf7d4334
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bf7d4334
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bf7d4334

Branch: refs/heads/cassandra-3.0
Commit: bf7d4334edf7f845f1abdc6ed2285a1738788b13
Parents: 251449f 3128802
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Tue Apr 26 08:28:33 2016 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Apr 26 08:28:33 2016 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../compaction/LeveledCompactionStrategy.java   | 15 ++++-
 .../db/compaction/LeveledManifest.java          | 15 ++++-
 .../LongLeveledCompactionStrategyTest.java      | 64 ++++++++++++++++++++
 4 files changed, 91 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ea21ee7,06b27b4..7d6760b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -23,25 -8,16 +23,26 @@@ Merged from 2.1
   * cqlsh COPY FROM fails with []{} chars in UDT/tuple fields/values (CASSANDRA-11633)
   * clqsh: COPY FROM throws TypeError with Cython extensions enabled (CASSANDRA-11574)
   * cqlsh: COPY FROM ignores NULL values in conversion (CASSANDRA-11549)
 - * (cqlsh) Fix potential COPY deadlock when parent process is terminating child
 -   processes (CASSANDRA-11505)
+  * Validate levels when building LeveledScanner to avoid overlaps with orphaned sstables
(CASSANDRA-9935)
  
  
 -2.2.6
 - * Allow only DISTINCT queries with partition keys restrictions (CASSANDRA-11339)
 - * CqlConfigHelper no longer requires both a keystore and truststore to work (CASSANDRA-11532)
 - * Make deprecated repair methods backward-compatible with previous notification service
(CASSANDRA-11430)
 - * IncomingStreamingConnection version check message wrong (CASSANDRA-11462)
 +3.0.5
 + * Fix rare NPE on schema upgrade from 2.x to 3.x (CASSANDRA-10943)
 + * Improve backoff policy for cqlsh COPY FROM (CASSANDRA-11320)
 + * Improve IF NOT EXISTS check in CREATE INDEX (CASSANDRA-11131)
 + * Upgrade ohc to 0.4.3
 + * Enable SO_REUSEADDR for JMX RMI server sockets (CASSANDRA-11093)
 + * Allocate merkletrees with the correct size (CASSANDRA-11390)
 + * Support streaming pre-3.0 sstables (CASSANDRA-10990)
 + * Add backpressure to compressed commit log (CASSANDRA-10971)
 + * SSTableExport supports secondary index tables (CASSANDRA-11330)
 + * Fix sstabledump to include missing info in debug output (CASSANDRA-11321)
 + * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331)
 + * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208)
 + * Remove recursive call from CompositesSearcher (CASSANDRA-11304)
 + * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377)
 + * Fix sstableloader fail when using materialized view (CASSANDRA-11275)
 +Merged from 2.2:
   * DatabaseDescriptor should log stacktrace in case of Eception during seed provider creation
(CASSANDRA-11312)
   * Use canonical path for directory in SSTable descriptor (CASSANDRA-10587)
   * Add cassandra-stress keystore option (CASSANDRA-9325)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
index 5a5cbef,bdc9141..cd74620
--- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java
@@@ -216,8 -215,10 +216,10 @@@ public class LeveledCompactionStrategy 
          return maxSSTableSizeInMB * 1024L * 1024L;
      }
  
 -    public ScannerList getScanners(Collection<SSTableReader> sstables, Range<Token>
range)
 +    public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>>
ranges)
      {
+         Set<SSTableReader>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot();
+ 
          Multimap<Integer, SSTableReader> byLevel = ArrayListMultimap.create();
          for (SSTableReader sstable : sstables)
          {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
index b0cd077,3b207da..2bfe88f
--- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
+++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java
@@@ -25,10 -25,9 +25,11 @@@ import com.google.common.base.Predicate
  import com.google.common.base.Predicates;
  import com.google.common.collect.ImmutableSortedSet;
  import com.google.common.collect.Iterables;
+ import com.google.common.collect.Lists;
  import com.google.common.collect.Sets;
  import com.google.common.primitives.Ints;
 +
 +import org.apache.cassandra.db.PartitionPosition;
  import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
index 96ee072,5439a72..79497aa
--- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
@@@ -21,19 -21,22 +21,26 @@@ import java.nio.ByteBuffer
  import java.util.*;
  import java.util.concurrent.*;
  
 -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
++import com.google.common.collect.Lists;
++
++import org.apache.cassandra.db.lifecycle.SSTableSet;
++import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+ import org.apache.cassandra.io.sstable.ISSTableScanner;
  import org.apache.cassandra.io.sstable.format.SSTableReader;
  import org.junit.BeforeClass;
  import org.junit.Test;
  
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
 -import org.apache.cassandra.config.KSMetaData;
 +import org.apache.cassandra.UpdateBuilder;
  import org.apache.cassandra.db.*;
  import org.apache.cassandra.exceptions.ConfigurationException;
 -import org.apache.cassandra.locator.SimpleStrategy;
 +import org.apache.cassandra.schema.CompactionParams;
 +import org.apache.cassandra.schema.KeyspaceParams;
  import org.apache.cassandra.utils.FBUtilities;
  
+ import static org.junit.Assert.assertTrue;
+ 
  public class LongLeveledCompactionStrategyTest
  {
      public static final String KEYSPACE1 = "LongLeveledCompactionStrategyTest";
@@@ -134,4 -140,62 +141,61 @@@
              }
          }
      }
+ 
+     @Test
+     public void testLeveledScanner() throws Exception
+     {
+         testParallelLeveledCompaction();
+         Keyspace keyspace = Keyspace.open(KEYSPACE1);
+         ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL);
+         store.disableAutoCompaction();
+ 
 -        WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy());
 -        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1);
++        LeveledCompactionStrategy lcs = (LeveledCompactionStrategy)store.getCompactionStrategyManager().getStrategies().get(1);
+ 
+         ByteBuffer value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value
+ 
+         // Adds 10 partitions
+         for (int r = 0; r < 10; r++)
+         {
+             DecoratedKey key = Util.dk(String.valueOf(r));
 -            Mutation rm = new Mutation(KEYSPACE1, key.getKey());
++            UpdateBuilder builder = UpdateBuilder.create(store.metadata, key);
+             for (int c = 0; c < 10; c++)
 -            {
 -                rm.add(CF_STANDARDLVL, Util.cellname("column" + c), value, 0);
 -            }
++                builder.newRow("column" + c).add("val", value);
++
++            Mutation rm = new Mutation(builder.build());
+             rm.apply();
+         }
+ 
+         //Flush sstable
+         store.forceBlockingFlush();
+ 
 -        Collection<SSTableReader> allSSTables = store.getSSTables();
++        Iterable<SSTableReader> allSSTables = store.getSSTables(SSTableSet.LIVE);
+         for (SSTableReader sstable : allSSTables)
+         {
+             if (sstable.getSSTableLevel() == 0)
+             {
+                 System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: "
+ sstable.getFilename());
+                 sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor,
1);
+                 sstable.reloadSSTableMetadata();
+             }
+         }
+ 
 -        try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables))
++        try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(Lists.newArrayList(allSSTables)))
+         {
+             //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935)
+             for (ISSTableScanner scanner : scannerList.scanners)
+             {
+                 DecoratedKey lastKey = null;
+                 while (scanner.hasNext())
+                 {
 -                    OnDiskAtomIterator row = scanner.next();
++                    UnfilteredRowIterator row = scanner.next();
+                     if (lastKey != null)
+                     {
 -                        assertTrue("row " + row.getKey() + " received out of order wrt "
+ lastKey, row.getKey().compareTo(lastKey) >= 0);
++                        assertTrue("row " + row.partitionKey() + " received out of order
wrt " + lastKey, row.partitionKey().compareTo(lastKey) >= 0);
+                     }
 -                    lastKey = row.getKey();
++                    lastKey = row.partitionKey();
+                 }
+             }
+         }
+     }
  }


Mime
View raw message