cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stefa...@apache.org
Subject [5/6] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
Date Fri, 08 Sep 2017 04:46:24 GMT
Merge branch 'cassandra-3.0' into cassandra-3.11


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/260475d0
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/260475d0
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/260475d0

Branch: refs/heads/cassandra-3.11
Commit: 260475d0f2b1f9a8344a3cfb38f6766c9241b52e
Parents: 5dcb354 ae88fd6
Author: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Authored: Fri Sep 8 12:43:30 2017 +0800
Committer: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Committed: Fri Sep 8 12:43:30 2017 +0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../io/sstable/IndexSummaryBuilder.java         |  70 ++++++++---
 .../cassandra/io/sstable/IndexSummaryTest.java  | 123 +++++++++++++++++++
 3 files changed, 179 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/260475d0/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 75d9bd6,f4360be..23e6c8b
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,14 -1,4 +1,15 @@@
 -3.0.15
 +3.11.1
 + * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418)
 + * BTree.Builder memory leak (CASSANDRA-13754)
 + * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798)
 + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938)
 + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744)
 + * "ignore" option is ignored in sstableloader (CASSANDRA-13721)
 + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652)
 + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512)
 + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641)
 +Merged from 3.0:
++ * Avoid assertion error when IndexSummary > 2G (CASSANDRA-12014)
   * Change repair midpoint logging for tiny ranges (CASSANDRA-13603)
   * Better handle corrupt final commitlog segment (CASSANDRA-11995)
   * StreamingHistogram is not thread safe (CASSANDRA-13756)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/260475d0/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
index f3757a0,ad08ba0..ab0486d
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
@@@ -45,14 -44,126 +45,137 @@@ import static org.junit.Assert.*
  
  public class IndexSummaryTest
  {
+     private final static Random random = new Random();
 -    private final static IPartitioner partitioner = Util.testPartitioner();
++
 +    @BeforeClass
 +    public static void initDD()
 +    {
 +        DatabaseDescriptor.daemonInitialization();
++
++        final long seed = System.nanoTime();
++        System.out.println("Using seed: " + seed);
++        random.setSeed(seed);
 +    }
 +
 +    IPartitioner partitioner = Util.testPartitioner();
  
+     @BeforeClass
+     public static void setup()
+     {
+         final long seed = System.nanoTime();
+         System.out.println("Using seed: " + seed);
+         random.setSeed(seed);
+     }
+ 
+     @Test
+     public void testIndexSummaryKeySizes() throws IOException
+     {
+         testIndexSummaryProperties(32, 100);
+         testIndexSummaryProperties(64, 100);
+         testIndexSummaryProperties(100, 100);
+         testIndexSummaryProperties(1000, 100);
+         testIndexSummaryProperties(10000, 100);
+     }
+ 
+     private void testIndexSummaryProperties(int keySize, int numKeys) throws IOException
+     {
+         final int minIndexInterval = 1;
+         final List<DecoratedKey> keys = new ArrayList<>(numKeys);
+ 
+         try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval,
BASE_SAMPLING_LEVEL))
+         {
+             for (int i = 0; i < numKeys; i++)
+             {
+                 byte[] randomBytes = new byte[keySize];
+                 random.nextBytes(randomBytes);
+                 DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+                 keys.add(key);
+                 builder.maybeAddEntry(key, i);
+             }
+ 
+             try(IndexSummary indexSummary = builder.build(partitioner))
+             {
+                 assertEquals(numKeys, keys.size());
+                 assertEquals(minIndexInterval, indexSummary.getMinIndexInterval());
+                 assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
+                 assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
+ 
+                 for (int i = 0; i < numKeys; i++)
+                     assertEquals(keys.get(i).getKey(), ByteBuffer.wrap(indexSummary.getKey(i)));
+             }
+         }
+     }
+ 
+     /**
+      * Test an index summary whose total size is bigger than 2GB,
+      * the index summary builder should log an error but it should still
+      * create an index summary, albeit one that does not cover the entire sstable.
+      */
+     @Test
+     public void tesLargeIndexSummary() throws IOException
+     {
+         final int numKeys = 1000000;
+         final int keySize = 3000;
+         final int minIndexInterval = 1;
+ 
+         try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval,
BASE_SAMPLING_LEVEL))
+         {
+             for (int i = 0; i < numKeys; i++)
+             {
+                 byte[] randomBytes = new byte[keySize];
+                 random.nextBytes(randomBytes);
+                 DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+                 builder.maybeAddEntry(key, i);
+             }
+ 
+             try (IndexSummary indexSummary = builder.build(partitioner))
+             {
+                 assertNotNull(indexSummary);
+                 assertEquals(numKeys, indexSummary.getMaxNumberOfEntries());
+                 assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount());
+             }
+         }
+     }
+ 
+     /**
+      * Test an index summary whose total size is bigger than 2GB,
+      * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size,
+      * the index summary should be downsampled automatically.
+      */
+     @Test
+     public void tesLargeIndexSummaryWithExpectedSizeMatching() throws IOException
+     {
+         final int numKeys = 1000000;
+         final int keySize = 3000;
+         final int minIndexInterval = 1;
+ 
+         long oldExpectedKeySize = IndexSummaryBuilder.defaultExpectedKeySize;
+         IndexSummaryBuilder.defaultExpectedKeySize = 3000;
+ 
+         try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval,
BASE_SAMPLING_LEVEL))
+         {
+             for (int i = 0; i < numKeys; i++)
+             {
+                 byte[] randomBytes = new byte[keySize];
+                 random.nextBytes(randomBytes);
+                 DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes));
+                 builder.maybeAddEntry(key, i);
+             }
+ 
+             try (IndexSummary indexSummary = builder.build(partitioner))
+             {
+                 assertNotNull(indexSummary);
+                 assertEquals(minIndexInterval * 2, indexSummary.getMinIndexInterval());
+                 assertEquals(numKeys / 2, indexSummary.getMaxNumberOfEntries());
+                 assertEquals(numKeys + 2, indexSummary.getEstimatedKeyCount());
+             }
+         }
+         finally
+         {
+             IndexSummaryBuilder.defaultExpectedKeySize = oldExpectedKeySize;
+         }
+     }
+ 
      @Test
      public void testGetKey()
      {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message