cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tylerho...@apache.org
Subject [2/2] cassandra git commit: Merge branch 'cassandra-2.2' into cassandra-3.0
Date Thu, 11 Feb 2016 23:55:02 GMT
Merge branch 'cassandra-2.2' into cassandra-3.0


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4e6f08d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4e6f08d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4e6f08d

Branch: refs/heads/cassandra-3.0
Commit: d4e6f08d48317796bfda8691f85c038fcd264769
Parents: 604c9df d5c83f4
Author: Tyler Hobbs <tylerlhobbs@gmail.com>
Authored: Thu Feb 11 17:54:22 2016 -0600
Committer: Tyler Hobbs <tylerlhobbs@gmail.com>
Committed: Thu Feb 11 17:54:22 2016 -0600

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../io/sstable/format/SSTableReader.java        |  13 ++-
 .../cassandra/io/sstable/SSTableReaderTest.java | 113 ++++++++++++++-----
 3 files changed, 93 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4e6f08d/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 7a0357f,fa25980..5156b0c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,15 -1,5 +1,16 @@@
 -2.2.6
 +3.0.4
 + * Properly handle hinted handoff after topology changes (CASSANDRA-5902)
 + * AssertionError when listing sstable files on inconsistent disk state (CASSANDRA-11156)
 + * Fix wrong rack counting and invalid conditions check for TokenAllocation
 +   (CASSANDRA-11139)
 + * Avoid creating empty hint files (CASSANDRA-11090)
 + * Fix leak detection strong reference loop using weak reference (CASSANDRA-11120)
 + * Configurie BatchlogManager to stop delayed tasks on shutdown (CASSANDRA-11062)
 + * Hadoop integration is incompatible with Cassandra Driver 3.0.0 (CASSANDRA-11001)
 + * Add dropped_columns to the list of schema table so it gets handled
 +   properly (CASSANDRA-11050)
 +Merged from 2.2:
+  * Always persist upsampled index summaries (CASSANDRA-10512)
   * (cqlsh) Fix inconsistent auto-complete (CASSANDRA-10733)
   * Make SELECT JSON and toJson() threadsafe (CASSANDRA-11048)
   * Fix SELECT on tuple relations for mixed ASC/DESC clustering order (CASSANDRA-7281)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4e6f08d/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index 1618516,e81e4e9..691bf45
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@@ -1175,13 -1154,7 +1175,7 @@@ public abstract class SSTableReader ext
              else if (samplingLevel < indexSummary.getSamplingLevel())
              {
                  // we can use the existing index summary to make a smaller one
 -                newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel,
minIndexInterval, partitioner);
 +                newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel,
minIndexInterval, getPartitioner());
- 
-                 try(SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode(),
false);
-                     SegmentedFile.Builder dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode(),
compression))
-                 {
-                     saveSummary(ibuilder, dbuilder, newSummary);
-                 }
              }
              else
              {
@@@ -1189,6 -1162,18 +1183,13 @@@
                          "no adjustments to min/max_index_interval");
              }
  
+             //Always save the resampled index
+             try(SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode(),
false);
+                 SegmentedFile.Builder dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode(),
compression))
+             {
 -                for (long boundry : dfile.copyReadableBounds())
 -                    dbuilder.addPotentialBoundary(boundry);
 -                for (long boundry : ifile.copyReadableBounds())
 -                    ibuilder.addPotentialBoundary(boundry);
 -
+                 saveSummary(ibuilder, dbuilder, newSummary);
+             }
+ 
              long newSize = bytesOnDisk();
              StorageMetrics.load.inc(newSize - oldSize);
              parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4e6f08d/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index c7f3c36,0000000..640b68b
mode 100644,000000..100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@@ -1,567 -1,0 +1,624 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.cassandra.io.sstable;
 +
 +import java.io.IOException;
 +import java.nio.ByteBuffer;
 +import java.util.*;
 +import java.util.concurrent.*;
 +
 +import com.google.common.collect.Sets;
 +import org.junit.Assert;
 +import org.junit.BeforeClass;
 +import org.junit.Test;
 +import org.junit.runner.RunWith;
 +
 +import org.apache.cassandra.OrderedJUnit4ClassRunner;
 +import org.apache.cassandra.SchemaLoader;
 +import org.apache.cassandra.Util;
 +import org.apache.cassandra.config.DatabaseDescriptor;
 +import org.apache.cassandra.cql3.Operator;
 +import org.apache.cassandra.db.*;
 +import org.apache.cassandra.db.compaction.CompactionManager;
 +import org.apache.cassandra.db.compaction.OperationType;
 +import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators;
 +import org.apache.cassandra.db.rows.Row;
 +import org.apache.cassandra.dht.IPartitioner;
 +import org.apache.cassandra.dht.LocalPartitioner.LocalToken;
 +import org.apache.cassandra.dht.Range;
 +import org.apache.cassandra.dht.Token;
 +import org.apache.cassandra.index.Index;
 +import org.apache.cassandra.io.sstable.format.SSTableReader;
 +import org.apache.cassandra.io.util.FileDataInput;
 +import org.apache.cassandra.io.util.MmappedRegions;
 +import org.apache.cassandra.io.util.MmappedSegmentedFile;
 +import org.apache.cassandra.io.util.SegmentedFile;
 +import org.apache.cassandra.schema.CachingParams;
 +import org.apache.cassandra.schema.KeyspaceParams;
 +import org.apache.cassandra.service.CacheService;
 +import org.apache.cassandra.utils.ByteBufferUtil;
 +import org.apache.cassandra.utils.Pair;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertNotNull;
 +import static org.junit.Assert.assertTrue;
 +
 +@RunWith(OrderedJUnit4ClassRunner.class)
 +public class SSTableReaderTest
 +{
 +    public static final String KEYSPACE1 = "SSTableReaderTest";
 +    public static final String CF_STANDARD = "Standard1";
 +    public static final String CF_STANDARD2 = "Standard2";
 +    public static final String CF_INDEXED = "Indexed1";
 +    public static final String CF_STANDARDLOWINDEXINTERVAL = "StandardLowIndexInterval";
 +
 +    private IPartitioner partitioner;
 +
 +    Token t(int i)
 +    {
 +        return partitioner.getToken(ByteBufferUtil.bytes(String.valueOf(i)));
 +    }
 +
 +    @BeforeClass
 +    public static void defineSchema() throws Exception
 +    {
 +        SchemaLoader.prepareServer();
 +        SchemaLoader.createKeyspace(KEYSPACE1,
 +                                    KeyspaceParams.simple(1),
 +                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD),
 +                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD2),
 +                                    SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED,
true),
 +                                    SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARDLOWINDEXINTERVAL)
 +                                                .minIndexInterval(8)
 +                                                .maxIndexInterval(256)
 +                                                .caching(CachingParams.CACHE_NOTHING));
 +    }
 +
 +    @Test
 +    public void testGetPositionsForRanges()
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
 +        partitioner = store.getPartitioner();
 +
 +        // insert data and compact to a single sstable
 +        CompactionManager.instance.disableAutoCompaction();
 +        for (int j = 0; j < 10; j++)
 +        {
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +                .clustering("0")
 +                .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +                .build()
 +                .applyUnsafe();
 +        }
 +        store.forceBlockingFlush();
 +        CompactionManager.instance.performMaximal(store, false);
 +
 +        List<Range<Token>> ranges = new ArrayList<Range<Token>>();
 +        // 1 key
 +        ranges.add(new Range<>(t(0), t(1)));
 +        // 2 keys
 +        ranges.add(new Range<>(t(2), t(4)));
 +        // wrapping range from key to end
 +        ranges.add(new Range<>(t(6), partitioner.getMinimumToken()));
 +        // empty range (should be ignored)
 +        ranges.add(new Range<>(t(9), t(91)));
 +
 +        // confirm that positions increase continuously
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        long previous = -1;
 +        for (Pair<Long,Long> section : sstable.getPositionsForRanges(ranges))
 +        {
 +            assert previous <= section.left : previous + " ! < " + section.left;
 +            assert section.left < section.right : section.left + " ! < " + section.right;
 +            previous = section.right;
 +        }
 +    }
 +
 +    @Test
 +    public void testSpannedIndexPositions() throws IOException
 +    {
++        int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
 +        MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this
will generate lots of segments
 +
-         Keyspace keyspace = Keyspace.open(KEYSPACE1);
-         ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
-         partitioner = store.getPartitioner();
- 
-         // insert a bunch of data and compact to a single sstable
-         CompactionManager.instance.disableAutoCompaction();
-         for (int j = 0; j < 100; j += 2)
++        try
 +        {
-             new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
-             .clustering("0")
-             .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
-             .build()
-             .applyUnsafe();
-         }
-         store.forceBlockingFlush();
-         CompactionManager.instance.performMaximal(store, false);
++            Keyspace keyspace = Keyspace.open(KEYSPACE1);
++            ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
++            partitioner = store.getPartitioner();
 +
-         // check that all our keys are found correctly
-         SSTableReader sstable = store.getLiveSSTables().iterator().next();
-         for (int j = 0; j < 100; j += 2)
-         {
-             DecoratedKey dk = Util.dk(String.valueOf(j));
-             FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ).position);
-             DecoratedKey keyInDisk = sstable.decorateKey(ByteBufferUtil.readWithShortLength(file));
-             assert keyInDisk.equals(dk) : String.format("%s != %s in %s", keyInDisk, dk,
file.getPath());
-         }
++            // insert a bunch of data and compact to a single sstable
++            CompactionManager.instance.disableAutoCompaction();
++            for (int j = 0; j < 100; j += 2)
++            {
++                new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
++                .clustering("0")
++                .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
++                .build()
++                .applyUnsafe();
++            }
++            store.forceBlockingFlush();
++            CompactionManager.instance.performMaximal(store, false);
++
++            // check that all our keys are found correctly
++            SSTableReader sstable = store.getLiveSSTables().iterator().next();
++            for (int j = 0; j < 100; j += 2)
++            {
++                DecoratedKey dk = Util.dk(String.valueOf(j));
++                FileDataInput file = sstable.getFileDataInput(sstable.getPosition(dk, SSTableReader.Operator.EQ).position);
++                DecoratedKey keyInDisk = sstable.decorateKey(ByteBufferUtil.readWithShortLength(file));
++                assert keyInDisk.equals(dk) : String.format("%s != %s in %s", keyInDisk,
dk, file.getPath());
++            }
 +
-         // check no false positives
-         for (int j = 1; j < 110; j += 2)
++            // check no false positives
++            for (int j = 1; j < 110; j += 2)
++            {
++                DecoratedKey dk = Util.dk(String.valueOf(j));
++                assert sstable.getPosition(dk, SSTableReader.Operator.EQ) == null;
++            }
++        }
++        finally
 +        {
-             DecoratedKey dk = Util.dk(String.valueOf(j));
-             assert sstable.getPosition(dk, SSTableReader.Operator.EQ) == null;
++            MmappedRegions.MAX_SEGMENT_SIZE = originalMaxSegmentSize;
 +        }
 +    }
 +
 +    @Test
 +    public void testPersistentStatistics()
 +    {
 +
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
 +        partitioner = store.getPartitioner();
 +
 +        for (int j = 0; j < 100; j += 2)
 +        {
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +            .clustering("0")
 +            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +            .build()
 +            .applyUnsafe();
 +        }
 +        store.forceBlockingFlush();
 +
 +        clearAndLoad(store);
 +        assert store.metric.maxPartitionSize.getValue() != 0;
 +    }
 +
 +    private void clearAndLoad(ColumnFamilyStore cfs)
 +    {
 +        cfs.clearUnsafe();
 +        cfs.loadNewSSTables();
 +    }
 +
 +    @Test
 +    public void testReadRateTracking()
 +    {
 +        // try to make sure CASSANDRA-8239 never happens again
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
 +        partitioner = store.getPartitioner();
 +
 +        for (int j = 0; j < 10; j++)
 +        {
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +            .clustering("0")
 +            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +            .build()
 +            .applyUnsafe();
 +        }
 +
 +        store.forceBlockingFlush();
 +
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        assertEquals(0, sstable.getReadMeter().count());
 +
 +        DecoratedKey key = sstable.decorateKey(ByteBufferUtil.bytes("4"));
 +        Util.getAll(Util.cmd(store, key).build());
 +        assertEquals(1, sstable.getReadMeter().count());
 +
 +        Util.getAll(Util.cmd(store, key).includeRow("0").build());
 +        assertEquals(2, sstable.getReadMeter().count());
 +    }
 +
 +    @Test
 +    public void testGetPositionsForRangesWithKeyCache()
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
 +        partitioner = store.getPartitioner();
 +        CacheService.instance.keyCache.setCapacity(100);
 +
 +        // insert data and compact to a single sstable
 +        CompactionManager.instance.disableAutoCompaction();
 +        for (int j = 0; j < 10; j++)
 +        {
 +
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +            .clustering("0")
 +            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +            .build()
 +            .applyUnsafe();
 +
 +        }
 +        store.forceBlockingFlush();
 +        CompactionManager.instance.performMaximal(store, false);
 +
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        long p2 = sstable.getPosition(k(2), SSTableReader.Operator.EQ).position;
 +        long p3 = sstable.getPosition(k(3), SSTableReader.Operator.EQ).position;
 +        long p6 = sstable.getPosition(k(6), SSTableReader.Operator.EQ).position;
 +        long p7 = sstable.getPosition(k(7), SSTableReader.Operator.EQ).position;
 +
 +        Pair<Long, Long> p = sstable.getPositionsForRanges(makeRanges(t(2), t(6))).get(0);
 +
 +        // range are start exclusive so we should start at 3
 +        assert p.left == p3;
 +
 +        // to capture 6 we have to stop at the start of 7
 +        assert p.right == p7;
 +    }
 +
 +    @Test
 +    public void testPersistentStatisticsWithSecondaryIndex()
 +    {
 +        // Create secondary index and flush to disk
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_INDEXED);
 +        partitioner = store.getPartitioner();
 +
 +        new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1")
 +            .clustering("0")
 +            .add("birthdate", 1L)
 +            .build()
 +            .applyUnsafe();
 +
 +        store.forceBlockingFlush();
 +
 +        // check if opening and querying works
 +        assertIndexQueryWorks(store);
 +    }
 +    public void testGetPositionsKeyCacheStats()
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
 +        partitioner = store.getPartitioner();
 +        CacheService.instance.keyCache.setCapacity(1000);
 +
 +        // insert data and compact to a single sstable
 +        CompactionManager.instance.disableAutoCompaction();
 +        for (int j = 0; j < 10; j++)
 +        {
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +            .clustering("0")
 +            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +            .build()
 +            .applyUnsafe();
 +        }
 +        store.forceBlockingFlush();
 +        CompactionManager.instance.performMaximal(store, false);
 +
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        sstable.getPosition(k(2), SSTableReader.Operator.EQ);
 +        assertEquals(0, sstable.getKeyCacheHit());
 +        assertEquals(1, sstable.getBloomFilterTruePositiveCount());
 +        sstable.getPosition(k(2), SSTableReader.Operator.EQ);
 +        assertEquals(1, sstable.getKeyCacheHit());
 +        assertEquals(2, sstable.getBloomFilterTruePositiveCount());
 +        sstable.getPosition(k(15), SSTableReader.Operator.EQ);
 +        assertEquals(1, sstable.getKeyCacheHit());
 +        assertEquals(2, sstable.getBloomFilterTruePositiveCount());
 +
 +    }
 +
 +
 +    @Test
 +    public void testOpeningSSTable() throws Exception
 +    {
 +        String ks = KEYSPACE1;
 +        String cf = "Standard1";
 +
 +        // clear and create just one sstable for this test
 +        Keyspace keyspace = Keyspace.open(ks);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore(cf);
 +        store.clearUnsafe();
 +        store.disableAutoCompaction();
 +
 +        DecoratedKey firstKey = null, lastKey = null;
 +        long timestamp = System.currentTimeMillis();
 +        for (int i = 0; i < store.metadata.params.minIndexInterval; i++)
 +        {
 +            DecoratedKey key = Util.dk(String.valueOf(i));
 +            if (firstKey == null)
 +                firstKey = key;
 +            if (lastKey == null)
 +                lastKey = key;
 +            if (store.metadata.getKeyValidator().compare(lastKey.getKey(), key.getKey())
< 0)
 +                lastKey = key;
 +
 +
 +            new RowUpdateBuilder(store.metadata, timestamp, key.getKey())
 +                .clustering("col")
 +                .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +                .build()
 +                .applyUnsafe();
 +
 +        }
 +        store.forceBlockingFlush();
 +
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        Descriptor desc = sstable.descriptor;
 +
 +        // test to see if sstable can be opened as expected
 +        SSTableReader target = SSTableReader.open(desc);
 +        Assert.assertEquals(target.getIndexSummarySize(), 1);
 +        Assert.assertArrayEquals(ByteBufferUtil.getArray(firstKey.getKey()), target.getIndexSummaryKey(0));
 +        assert target.first.equals(firstKey);
 +        assert target.last.equals(lastKey);
 +        target.selfRef().release();
 +    }
 +
 +    @Test
 +    public void testLoadingSummaryUsesCorrectPartitioner() throws Exception
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Indexed1");
 +
 +        new RowUpdateBuilder(store.metadata, System.currentTimeMillis(), "k1")
 +        .clustering("0")
 +        .add("birthdate", 1L)
 +        .build()
 +        .applyUnsafe();
 +
 +        store.forceBlockingFlush();
 +
 +        for(ColumnFamilyStore indexCfs : store.indexManager.getAllIndexColumnFamilyStores())
 +        {
 +            assert indexCfs.isIndex();
 +            SSTableReader sstable = indexCfs.getLiveSSTables().iterator().next();
 +            assert sstable.first.getToken() instanceof LocalToken;
 +
 +            try (SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode(),
 +                                                                           false);
 +                 SegmentedFile.Builder dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode(),
 +                                                                           sstable.compression))
 +            {
 +                sstable.saveSummary(ibuilder, dbuilder);
 +            }
 +            SSTableReader reopened = SSTableReader.open(sstable.descriptor);
 +            assert reopened.first.getToken() instanceof LocalToken;
 +            reopened.selfRef().release();
 +        }
 +    }
 +
 +    /** see CASSANDRA-5407 */
 +    @Test
 +    public void testGetScannerForNoIntersectingRanges() throws Exception
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
 +        partitioner = store.getPartitioner();
 +
 +        new RowUpdateBuilder(store.metadata, 0, "k1")
 +            .clustering("xyz")
 +            .add("val", "abc")
 +            .build()
 +            .applyUnsafe();
 +
 +        store.forceBlockingFlush();
 +        boolean foundScanner = false;
 +        for (SSTableReader s : store.getLiveSSTables())
 +        {
 +            try (ISSTableScanner scanner = s.getScanner(new Range<Token>(t(0), t(1)),
null))
 +            {
 +                scanner.next(); // throws exception pre 5407
 +                foundScanner = true;
 +            }
 +        }
 +        assertTrue(foundScanner);
 +    }
 +
 +    @Test
 +    public void testGetPositionsForRangesFromTableOpenedForBulkLoading() throws IOException
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
 +        partitioner = store.getPartitioner();
 +
 +        // insert data and compact to a single sstable. The
 +        // number of keys inserted is greater than index_interval
 +        // to ensure multiple segments in the index file
 +        CompactionManager.instance.disableAutoCompaction();
 +        for (int j = 0; j < 130; j++)
 +        {
 +
 +            new RowUpdateBuilder(store.metadata, j, String.valueOf(j))
 +            .clustering("0")
 +            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
 +            .build()
 +            .applyUnsafe();
 +
 +        }
 +        store.forceBlockingFlush();
 +        CompactionManager.instance.performMaximal(store, false);
 +
 +        // construct a range which is present in the sstable, but whose
 +        // keys are not found in the first segment of the index.
 +        List<Range<Token>> ranges = new ArrayList<Range<Token>>();
 +        ranges.add(new Range<Token>(t(98), t(99)));
 +
 +        SSTableReader sstable = store.getLiveSSTables().iterator().next();
 +        List<Pair<Long,Long>> sections = sstable.getPositionsForRanges(ranges);
 +        assert sections.size() == 1 : "Expected to find range in sstable" ;
 +
 +        // re-open the same sstable as it would be during bulk loading
 +        Set<Component> components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);
 +        if (sstable.components.contains(Component.COMPRESSION_INFO))
 +            components.add(Component.COMPRESSION_INFO);
 +        SSTableReader bulkLoaded = SSTableReader.openForBatch(sstable.descriptor, components,
store.metadata);
 +        sections = bulkLoaded.getPositionsForRanges(ranges);
 +        assert sections.size() == 1 : "Expected to find range in sstable opened for bulk
loading";
 +        bulkLoaded.selfRef().release();
 +    }
 +
 +    @Test
 +    public void testIndexSummaryReplacement() throws IOException, ExecutionException, InterruptedException
 +    {
 +        Keyspace keyspace = Keyspace.open(KEYSPACE1);
 +        final ColumnFamilyStore store = keyspace.getColumnFamilyStore("StandardLowIndexInterval");
// index interval of 8, no key caching
 +        CompactionManager.instance.disableAutoCompaction();
 +
 +        final int NUM_PARTITIONS = 512;
 +        for (int j = 0; j < NUM_PARTITIONS; j++)
 +        {
 +            new RowUpdateBuilder(store.metadata, j, String.format("%3d", j))
 +            .clustering("0")
 +            .add("val", String.format("%3d", j))
 +            .build()
 +            .applyUnsafe();
 +
 +        }
 +        store.forceBlockingFlush();
 +        CompactionManager.instance.performMaximal(store, false);
 +
 +        Collection<SSTableReader> sstables = store.getLiveSSTables();
 +        assert sstables.size() == 1;
 +        final SSTableReader sstable = sstables.iterator().next();
 +
 +        ThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(5);
 +        List<Future> futures = new ArrayList<>(NUM_PARTITIONS * 2);
 +        for (int i = 0; i < NUM_PARTITIONS; i++)
 +        {
 +            final ByteBuffer key = ByteBufferUtil.bytes(String.format("%3d", i));
 +            final int index = i;
 +
 +            futures.add(executor.submit(new Runnable()
 +            {
 +                public void run()
 +                {
 +                    Row row = Util.getOnlyRowUnfiltered(Util.cmd(store, key).build());
 +                    assertEquals(0, ByteBufferUtil.compare(String.format("%3d", index).getBytes(),
row.cells().iterator().next().value()));
 +                }
 +            }));
 +
 +            futures.add(executor.submit(new Runnable()
 +            {
 +                public void run()
 +                {
 +                    Iterable<DecoratedKey> results = store.keySamples(
 +                            new Range<>(sstable.getPartitioner().getMinimumToken(),
sstable.getPartitioner().getToken(key)));
 +                    assertTrue(results.iterator().hasNext());
 +                }
 +            }));
 +        }
 +
 +        SSTableReader replacement;
 +        try (LifecycleTransaction txn = store.getTracker().tryModify(Arrays.asList(sstable),
OperationType.UNKNOWN))
 +        {
 +            replacement = sstable.cloneWithNewSummarySamplingLevel(store, 1);
 +            txn.update(replacement, true);
 +            txn.finish();
 +        }
 +        for (Future future : futures)
 +            future.get();
 +
 +        assertEquals(sstable.estimatedKeys(), replacement.estimatedKeys(), 1);
 +    }
 +
++    @Test
++    public void testIndexSummaryUpsampleAndReload() throws Exception
++    {
++        int originalMaxSegmentSize = MmappedRegions.MAX_SEGMENT_SIZE;
++        MmappedRegions.MAX_SEGMENT_SIZE = 40; // each index entry is ~11 bytes, so this
will generate lots of segments
++
++        try
++        {
++            testIndexSummaryUpsampleAndReload0();
++        }
++        finally
++        {
++            MmappedRegions.MAX_SEGMENT_SIZE = originalMaxSegmentSize;
++        }
++    }
++
++    private void testIndexSummaryUpsampleAndReload0() throws Exception
++    {
++        Keyspace keyspace = Keyspace.open(KEYSPACE1);
++        final ColumnFamilyStore store = keyspace.getColumnFamilyStore("StandardLowIndexInterval");
// index interval of 8, no key caching
++        CompactionManager.instance.disableAutoCompaction();
++
++        final int NUM_PARTITIONS = 512;
++        for (int j = 0; j < NUM_PARTITIONS; j++)
++        {
++            new RowUpdateBuilder(store.metadata, j, String.format("%3d", j))
++            .clustering("0")
++            .add("val", String.format("%3d", j))
++            .build()
++            .applyUnsafe();
++
++        }
++        store.forceBlockingFlush();
++        CompactionManager.instance.performMaximal(store, false);
++
++        Collection<SSTableReader> sstables = store.getLiveSSTables();
++        assert sstables.size() == 1;
++        final SSTableReader sstable = sstables.iterator().next();
++
++        try (LifecycleTransaction txn = store.getTracker().tryModify(Arrays.asList(sstable),
OperationType.UNKNOWN))
++        {
++            SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(store,
sstable.getIndexSummarySamplingLevel() + 1);
++            txn.update(replacement, true);
++            txn.finish();
++        }
++        SSTableReader reopen = SSTableReader.open(sstable.descriptor);
++        assert reopen.getIndexSummarySamplingLevel() == sstable.getIndexSummarySamplingLevel()
+ 1;
++    }
++
 +    private void assertIndexQueryWorks(ColumnFamilyStore indexedCFS)
 +    {
 +        assert "Indexed1".equals(indexedCFS.name);
 +
 +        // make sure all sstables including 2ary indexes load from disk
 +        for (ColumnFamilyStore cfs : indexedCFS.concatWithIndexes())
 +            clearAndLoad(cfs);
 +
 +
 +        // query using index to see if sstable for secondary index opens
 +        ReadCommand rc = Util.cmd(indexedCFS).fromKeyIncl("k1").toKeyIncl("k3")
 +                                             .columns("birthdate")
 +                                             .filterOn("birthdate", Operator.EQ, 1L)
 +                                             .build();
 +        Index.Searcher searcher = indexedCFS.indexManager.getBestIndexFor(rc).searcherFor(rc);
 +        assertNotNull(searcher);
 +        try (ReadOrderGroup orderGroup = ReadOrderGroup.forCommand(rc))
 +        {
 +            assertEquals(1, Util.size(UnfilteredPartitionIterators.filter(searcher.search(orderGroup),
rc.nowInSec())));
 +        }
 +    }
 +
 +    private List<Range<Token>> makeRanges(Token left, Token right)
 +    {
 +        return Arrays.asList(new Range<>(left, right));
 +    }
 +
 +    private DecoratedKey k(int i)
 +    {
 +        return new BufferDecoratedKey(t(i), ByteBufferUtil.bytes(String.valueOf(i)));
 +    }
 +}


Mime
View raw message