cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alek...@apache.org
Subject [1/2] Replace index_interval with min/max_index_interval
Date Thu, 13 Feb 2014 22:35:49 GMT
Updated Branches:
  refs/heads/trunk e9f8fc716 -> ee477cc4d


http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
index 6d8712a..e32d8db 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummary.java
@@ -51,24 +51,31 @@ public class IndexSummary implements Closeable
     private static final Logger logger = LoggerFactory.getLogger(IndexSummary.class);
 
     public static final IndexSummarySerializer serializer = new IndexSummarySerializer();
-    private final int indexInterval;
+
+    /**
+     * A lower bound for the average number of partitions in between each index summary entry. A lower value means
+     * that more partitions will have an entry in the index summary when at the full sampling level.
+     */
+    private final int minIndexInterval;
+
     private final IPartitioner partitioner;
     private final int summarySize;
     private final int sizeAtFullSampling;
     private final Memory bytes;
 
     /**
-     * A value between MIN_SAMPLING_LEVEL and BASE_SAMPLING_LEVEL that represents how many of the original
+     * A value between 1 and BASE_SAMPLING_LEVEL that represents how many of the original
      * index summary entries ((1 / indexInterval) * numKeys) have been retained.
      *
      * Thus, this summary contains (samplingLevel / BASE_SAMPLING_LEVEL) * ((1 / indexInterval) * numKeys)) entries.
      */
     private final int samplingLevel;
 
-    public IndexSummary(IPartitioner partitioner, Memory memory, int summarySize, int sizeAtFullSampling, int indexInterval, int samplingLevel)
+    public IndexSummary(IPartitioner partitioner, Memory memory, int summarySize, int sizeAtFullSampling,
+                        int minIndexInterval, int samplingLevel)
     {
         this.partitioner = partitioner;
-        this.indexInterval = indexInterval;
+        this.minIndexInterval = minIndexInterval;
         this.summarySize = summarySize;
         this.sizeAtFullSampling = sizeAtFullSampling;
         this.bytes = memory;
@@ -140,9 +147,22 @@ public class IndexSummary implements Closeable
         return index == (summarySize - 1) ? bytes.size() : getPositionInSummary(index + 1);
     }
 
-    public int getIndexInterval()
+    public int getMinIndexInterval()
     {
-        return indexInterval;
+        return minIndexInterval;
+    }
+
+    public double getEffectiveIndexInterval()
+    {
+        return (BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval;
+    }
+
+    /**
+     * Returns an estimate of the total number of keys in the SSTable.
+     */
+    public long getEstimatedKeyCount()
+    {
+        return ((long) getMaxNumberOfEntries() + 1) * minIndexInterval;
     }
 
     public int size()
@@ -157,7 +177,7 @@ public class IndexSummary implements Closeable
 
     /**
      * Returns the number of entries this summary would have if it were at the full sampling level, which is equal
-     * to the number of entries in the primary on-disk index divided by the index interval.
+     * to the number of entries in the primary on-disk index divided by the min index interval.
      */
     public int getMaxNumberOfEntries()
     {
@@ -184,14 +204,14 @@ public class IndexSummary implements Closeable
      */
     public int getEffectiveIndexIntervalAfterIndex(int index)
     {
-        return Downsampling.getEffectiveIndexIntervalAfterIndex(index, samplingLevel, indexInterval);
+        return Downsampling.getEffectiveIndexIntervalAfterIndex(index, samplingLevel, minIndexInterval);
     }
 
     public static class IndexSummarySerializer
     {
         public void serialize(IndexSummary t, DataOutputStream out, boolean withSamplingLevel) throws IOException
         {
-            out.writeInt(t.indexInterval);
+            out.writeInt(t.minIndexInterval);
             out.writeInt(t.summarySize);
             out.writeLong(t.bytes.size());
             if (withSamplingLevel)
@@ -202,14 +222,15 @@ public class IndexSummary implements Closeable
             FBUtilities.copy(new MemoryInputStream(t.bytes), out, t.bytes.size());
         }
 
-        public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel, int expectedIndexInterval) throws IOException
+        public IndexSummary deserialize(DataInputStream in, IPartitioner partitioner, boolean haveSamplingLevel, int expectedMinIndexInterval, int maxIndexInterval) throws IOException
         {
-            int indexInterval = in.readInt();
-            if (indexInterval != expectedIndexInterval)
+            int minIndexInterval = in.readInt();
+            if (minIndexInterval != expectedMinIndexInterval)
             {
-                throw new IOException(String.format("Cannot read index summary because Index Interval changed from %d to %d.",
-                                                           indexInterval, expectedIndexInterval));
+                throw new IOException(String.format("Cannot read index summary because min_index_interval changed from %d to %d.",
+                                                    minIndexInterval, expectedMinIndexInterval));
             }
+
             int summarySize = in.readInt();
             long offheapSize = in.readLong();
             int samplingLevel, fullSamplingSummarySize;
@@ -223,9 +244,17 @@ public class IndexSummary implements Closeable
                 samplingLevel = BASE_SAMPLING_LEVEL;
                 fullSamplingSummarySize = summarySize;
             }
+
+            int effectiveIndexInterval = (int) Math.ceil((BASE_SAMPLING_LEVEL / (double) samplingLevel) * minIndexInterval);
+            if (effectiveIndexInterval > maxIndexInterval)
+            {
+                throw new IOException(String.format("Rebuilding index summary because the effective index interval (%d) is higher than" +
+                                                    " the current max index interval (%d)", effectiveIndexInterval, maxIndexInterval));
+            }
+
             Memory memory = Memory.allocate(offheapSize);
             FBUtilities.copy(in, new MemoryOutputStream(memory), offheapSize);
-            return new IndexSummary(partitioner, memory, summarySize, fullSamplingSummarySize, indexInterval, samplingLevel);
+            return new IndexSummary(partitioner, memory, summarySize, fullSamplingSummarySize, minIndexInterval, samplingLevel);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
index 3635e7e..d77e887 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryBuilder.java
@@ -28,7 +28,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.io.sstable.Downsampling.MIN_SAMPLING_LEVEL;
 
 public class IndexSummaryBuilder
 {
@@ -36,43 +35,44 @@ public class IndexSummaryBuilder
 
     private final ArrayList<Long> positions;
     private final ArrayList<byte[]> keys;
-    private final int indexInterval;
+    private final int minIndexInterval;
     private final int samplingLevel;
     private final int[] startPoints;
     private long keysWritten = 0;
     private long indexIntervalMatches = 0;
     private long offheapSize = 0;
 
-    public IndexSummaryBuilder(long expectedKeys, int indexInterval, int samplingLevel)
+    public IndexSummaryBuilder(long expectedKeys, int minIndexInterval, int samplingLevel)
     {
-        this.indexInterval = indexInterval;
         this.samplingLevel = samplingLevel;
         this.startPoints = Downsampling.getStartPoints(BASE_SAMPLING_LEVEL, samplingLevel);
 
-        long expectedEntries = expectedKeys / indexInterval;
-        if (expectedEntries > Integer.MAX_VALUE)
+        long maxExpectedEntries = expectedKeys / minIndexInterval;
+        if (maxExpectedEntries > Integer.MAX_VALUE)
         {
-            // that's a _lot_ of keys, and a very low interval
-            int effectiveInterval = (int) Math.ceil((double) Integer.MAX_VALUE / expectedKeys);
-            expectedEntries = expectedKeys / effectiveInterval;
-            assert expectedEntries <= Integer.MAX_VALUE : expectedEntries;
-            logger.warn("Index interval of {} is too low for {} expected keys; using interval of {} instead",
-                        indexInterval, expectedKeys, effectiveInterval);
+            // that's a _lot_ of keys, and a very low min index interval
+            int effectiveMinInterval = (int) Math.ceil((double) Integer.MAX_VALUE / expectedKeys);
+            maxExpectedEntries = expectedKeys / effectiveMinInterval;
+            assert maxExpectedEntries <= Integer.MAX_VALUE : maxExpectedEntries;
+            logger.warn("min_index_interval of {} is too low for {} expected keys; using interval of {} instead",
+                        minIndexInterval, expectedKeys, effectiveMinInterval);
+            this.minIndexInterval = effectiveMinInterval;
+        }
+        else
+        {
+            this.minIndexInterval = minIndexInterval;
         }
 
-        // adjust our estimates based on the sampling level
-        expectedEntries = (expectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL;
-
-        positions = new ArrayList<>((int)expectedEntries);
-        keys = new ArrayList<>((int)expectedEntries);
+        // for initializing data structures, adjust our estimates based on the sampling level
+        maxExpectedEntries = (maxExpectedEntries * samplingLevel) / BASE_SAMPLING_LEVEL;
+        positions = new ArrayList<>((int)maxExpectedEntries);
+        keys = new ArrayList<>((int)maxExpectedEntries);
     }
 
     public IndexSummaryBuilder maybeAddEntry(DecoratedKey decoratedKey, long indexPosition)
     {
-        if (keysWritten % indexInterval == 0)
+        if (keysWritten % minIndexInterval == 0)
         {
-            indexIntervalMatches++;
-
             // see if we should skip this key based on our sampling level
             boolean shouldSkip = false;
             for (int start : startPoints)
@@ -92,6 +92,8 @@ public class IndexSummaryBuilder
                 positions.add(indexPosition);
                 offheapSize += TypeSizes.NATIVE.sizeof(indexPosition);
             }
+
+            indexIntervalMatches++;
         }
         keysWritten++;
 
@@ -124,17 +126,22 @@ public class IndexSummaryBuilder
             memory.setLong(keyPosition, actualIndexPosition);
             keyPosition += TypeSizes.NATIVE.sizeof(actualIndexPosition);
         }
-        int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) indexInterval);
-        return new IndexSummary(partitioner, memory, keys.size(), sizeAtFullSampling, indexInterval, samplingLevel);
+        int sizeAtFullSampling = (int) Math.ceil(keysWritten / (double) minIndexInterval);
+        return new IndexSummary(partitioner, memory, keys.size(), sizeAtFullSampling, minIndexInterval, samplingLevel);
     }
 
     public static int entriesAtSamplingLevel(int samplingLevel, int maxSummarySize)
     {
-        return (samplingLevel * maxSummarySize) / BASE_SAMPLING_LEVEL;
+        return (int) Math.ceil((samplingLevel * maxSummarySize) / (double) BASE_SAMPLING_LEVEL);
     }
 
-    public static int calculateSamplingLevel(int currentSamplingLevel, int currentNumEntries, long targetNumEntries)
+    public static int calculateSamplingLevel(int currentSamplingLevel, int currentNumEntries, long targetNumEntries, int minIndexInterval, int maxIndexInterval)
     {
+        // effective index interval == (BASE_SAMPLING_LEVEL / samplingLevel) * minIndexInterval
+        // so we can just solve for minSamplingLevel here:
+        // maxIndexInterval == (BASE_SAMPLING_LEVEL / minSamplingLevel) * minIndexInterval
+        int effectiveMinSamplingLevel = Math.max(1, (int) Math.ceil((BASE_SAMPLING_LEVEL * minIndexInterval) / (double) maxIndexInterval));
+
         // Algebraic explanation for calculating the new sampling level (solve for newSamplingLevel):
         // originalNumEntries = (baseSamplingLevel / currentSamplingLevel) * currentNumEntries
         // newSpaceUsed = (newSamplingLevel / baseSamplingLevel) * originalNumEntries
@@ -142,7 +149,7 @@ public class IndexSummaryBuilder
         // newSpaceUsed = (newSamplingLevel / currentSamplingLevel) * currentNumEntries
         // (newSpaceUsed * currentSamplingLevel) / currentNumEntries = newSamplingLevel
         int newSamplingLevel = (int) (targetNumEntries * currentSamplingLevel) / currentNumEntries;
-        return Math.min(BASE_SAMPLING_LEVEL, Math.max(MIN_SAMPLING_LEVEL, newSamplingLevel));
+        return Math.min(BASE_SAMPLING_LEVEL, Math.max(effectiveMinSamplingLevel, newSamplingLevel));
     }
 
     /**
@@ -153,7 +160,7 @@ public class IndexSummaryBuilder
      * @param partitioner the partitioner used for the index summary
      * @return a new IndexSummary
      */
-    public static IndexSummary downsample(IndexSummary existing, int newSamplingLevel, IPartitioner partitioner)
+    public static IndexSummary downsample(IndexSummary existing, int newSamplingLevel, int minIndexInterval, IPartitioner partitioner)
     {
         // To downsample the old index summary, we'll go through (potentially) several rounds of downsampling.
         // Conceptually, each round starts at position X and then removes every Nth item.  The value of X follows
@@ -162,6 +169,7 @@ public class IndexSummaryBuilder
 
         int currentSamplingLevel = existing.getSamplingLevel();
         assert currentSamplingLevel > newSamplingLevel;
+        assert minIndexInterval == existing.getMinIndexInterval();
 
         // calculate starting indexes for downsampling rounds
         int[] startPoints = Downsampling.getStartPoints(currentSamplingLevel, newSamplingLevel);
@@ -207,6 +215,7 @@ public class IndexSummaryBuilder
             memory.setBytes(keyPosition, entry, 0, entry.length);
             keyPosition += entry.length;
         }
-        return new IndexSummary(partitioner, memory, newKeyCount, existing.getMaxNumberOfEntries(), existing.getIndexInterval(), newSamplingLevel);
+        return new IndexSummary(partitioner, memory, newKeyCount, existing.getMaxNumberOfEntries(),
+                                minIndexInterval, newSamplingLevel);
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
index 1b36baf..b35f5f4 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManager.java
@@ -29,6 +29,7 @@ import javax.management.ObjectName;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.*;
 
+import org.apache.cassandra.config.CFMetaData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,6 +41,8 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.WrappedRunnable;
 
+import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
+
 /**
  * Manages the fixed-size memory pool for index summaries, periodically resizing them
  * in order to give more memory to hot sstables and less memory to cold sstables.
@@ -146,22 +149,22 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
         return memoryPoolBytes / 1024L / 1024L;
     }
 
-    public Map<String, Double> getSamplingRatios()
+    public Map<String, Integer> getIndexIntervals()
     {
         List<SSTableReader> sstables = getAllSSTables();
-        Map<String, Double> ratios = new HashMap<>(sstables.size());
+        Map<String, Integer> intervals = new HashMap<>(sstables.size());
         for (SSTableReader sstable : sstables)
-            ratios.put(sstable.getFilename(), sstable.getIndexSummarySamplingLevel() / (double) Downsampling.BASE_SAMPLING_LEVEL);
+            intervals.put(sstable.getFilename(), (int) Math.round(sstable.getEffectiveIndexInterval()));
 
-        return ratios;
+        return intervals;
     }
 
-    public double getAverageSamplingRatio()
+    public double getAverageIndexInterval()
     {
         List<SSTableReader> sstables = getAllSSTables();
         double total = 0.0;
         for (SSTableReader sstable : sstables)
-            total += sstable.getIndexSummarySamplingLevel() / (double) Downsampling.BASE_SAMPLING_LEVEL;
+            total += sstable.getEffectiveIndexInterval();
         return total / sstables.size();
     }
 
@@ -302,6 +305,8 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
 
         List<ResampleEntry> toDownsample = new ArrayList<>(sstables.size() / 4);
         List<ResampleEntry> toUpsample = new ArrayList<>(sstables.size() / 4);
+        List<ResampleEntry> forceResample = new ArrayList<>();
+        List<ResampleEntry> forceUpsample = new ArrayList<>();
         List<SSTableReader> newSSTables = new ArrayList<>(sstables.size());
 
         // Going from the coldest to the hottest sstables, try to give each sstable an amount of space proportional
@@ -309,6 +314,9 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
         long remainingSpace = memoryPoolCapacity;
         for (SSTableReader sstable : sstables)
         {
+            int minIndexInterval = sstable.metadata.getMinIndexInterval();
+            int maxIndexInterval = sstable.metadata.getMaxIndexInterval();
+
             double readsPerSec = sstable.readMeter == null ? 0.0 : sstable.readMeter.fifteenMinuteRate();
             long idealSpace = Math.round(remainingSpace * (readsPerSec / totalReadsPerSec));
 
@@ -317,14 +325,50 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
             double avgEntrySize = sstable.getIndexSummaryOffHeapSize() / (double) currentNumEntries;
             long targetNumEntries = Math.max(1, Math.round(idealSpace / avgEntrySize));
             int currentSamplingLevel = sstable.getIndexSummarySamplingLevel();
-            int newSamplingLevel = IndexSummaryBuilder.calculateSamplingLevel(currentSamplingLevel, currentNumEntries, targetNumEntries);
-            int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getMaxIndexSummarySize());
+            int maxSummarySize = sstable.getMaxIndexSummarySize();
 
-            logger.trace("{} has {} reads/sec; ideal space for index summary: {} bytes ({} entries); considering moving from level " +
-                         "{} ({} entries) to level {} ({} entries)",
-                         sstable.getFilename(), readsPerSec, idealSpace, targetNumEntries, currentSamplingLevel, currentNumEntries, newSamplingLevel, numEntriesAtNewSamplingLevel);
+            // if the min_index_interval changed, calculate what our current sampling level would be under the new min
+            if (sstable.getMinIndexInterval() != minIndexInterval)
+            {
+                int effectiveSamplingLevel = (int) Math.round(currentSamplingLevel * (minIndexInterval / (double) sstable.getMinIndexInterval()));
+                maxSummarySize = (int) Math.round(maxSummarySize * (sstable.getMinIndexInterval() / (double) minIndexInterval));
+                logger.trace("min_index_interval changed from {} to {}, so the current sampling level for {} is effectively now {} (was {})",
+                             sstable.getMinIndexInterval(), minIndexInterval, sstable, effectiveSamplingLevel, currentSamplingLevel);
+                currentSamplingLevel = effectiveSamplingLevel;
+            }
 
-            if (targetNumEntries >= currentNumEntries * UPSAMPLE_THRESHOLD && newSamplingLevel > currentSamplingLevel)
+            int newSamplingLevel = IndexSummaryBuilder.calculateSamplingLevel(currentSamplingLevel, currentNumEntries, targetNumEntries,
+                    minIndexInterval, maxIndexInterval);
+            int numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, maxSummarySize);
+            double effectiveIndexInterval = sstable.getEffectiveIndexInterval();
+
+            logger.trace("{} has {} reads/sec; ideal space for index summary: {} bytes ({} entries); considering moving " +
+                    "from level {} ({} entries, {} bytes) to level {} ({} entries, {} bytes)",
+                    sstable.getFilename(), readsPerSec, idealSpace, targetNumEntries, currentSamplingLevel, currentNumEntries,
+                    currentNumEntries * avgEntrySize, newSamplingLevel, numEntriesAtNewSamplingLevel,
+                    numEntriesAtNewSamplingLevel * avgEntrySize);
+
+            if (effectiveIndexInterval < minIndexInterval)
+            {
+                // The min_index_interval was changed; re-sample to match it.
+                logger.debug("Forcing resample of {} because the current index interval ({}) is below min_index_interval ({})",
+                        sstable, effectiveIndexInterval, minIndexInterval);
+                long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+                forceResample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
+                remainingSpace -= spaceUsed;
+            }
+            else if (effectiveIndexInterval > maxIndexInterval)
+            {
+                // The max_index_interval was lowered; force an upsample to the effective minimum sampling level
+                logger.debug("Forcing upsample of {} because the current index interval ({}) is above max_index_interval ({})",
+                        sstable, effectiveIndexInterval, maxIndexInterval);
+                newSamplingLevel = Math.max(1, (BASE_SAMPLING_LEVEL * minIndexInterval) / maxIndexInterval);
+                numEntriesAtNewSamplingLevel = IndexSummaryBuilder.entriesAtSamplingLevel(newSamplingLevel, sstable.getMaxIndexSummarySize());
+                long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
+                forceUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
+                remainingSpace -= avgEntrySize * numEntriesAtNewSamplingLevel;
+            }
+            else if (targetNumEntries >= currentNumEntries * UPSAMPLE_THRESHOLD && newSamplingLevel > currentSamplingLevel)
             {
                 long spaceUsed = (long) Math.ceil(avgEntrySize * numEntriesAtNewSamplingLevel);
                 toUpsample.add(new ResampleEntry(sstable, spaceUsed, newSamplingLevel));
@@ -354,7 +398,9 @@ public class IndexSummaryManager implements IndexSummaryManagerMBean
         }
 
         // downsample first, then upsample
+        toDownsample.addAll(forceResample);
         toDownsample.addAll(toUpsample);
+        toDownsample.addAll(forceUpsample);
         Multimap<DataTracker, SSTableReader> replacedByTracker = HashMultimap.create();
         Multimap<DataTracker, SSTableReader> replacementsByTracker = HashMultimap.create();
         for (ResampleEntry entry : toDownsample)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
index 1e115cb..3382350 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexSummaryManagerMBean.java
@@ -32,14 +32,11 @@ public interface IndexSummaryManagerMBean
     public double getMemoryPoolSizeInMB();
 
     /**
-     * Returns a map of SSTable filenames to their current sampling ratio, where 1.0 indicates that all of the
-     * original index summary entries have been retained and 0.5 indicates that half of the original entries have
-     * been discarded.
-     * @return A map of SSTable filenames to their sampling ratios.
+     * Returns a map of SSTable filenames to their current effective index interval.
      */
-    public Map<String, Double> getSamplingRatios();
+    public Map<String, Integer> getIndexIntervals();
 
-    public double getAverageSamplingRatio();
+    public double getAverageIndexInterval();
 
     public void redistributeSummaries() throws IOException;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
index ffb7be1..c02e397 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
@@ -192,10 +192,7 @@ public class SSTableReader extends SSTable implements Closeable
         if (count < 0)
         {
             for (SSTableReader sstable : sstables)
-            {
-                // using getMaxIndexSummarySize() lets us ignore the current sampling level
-                count += (sstable.getMaxIndexSummarySize() + 1) * sstable.indexSummary.getSamplingLevel();
-            }
+                count += sstable.estimatedKeys();
         }
         return count;
     }
@@ -608,7 +605,7 @@ public class SSTableReader extends SSTable implements Closeable
 
             IndexSummaryBuilder summaryBuilder = null;
             if (!summaryLoaded)
-                summaryBuilder = new IndexSummaryBuilder(estimatedKeys, metadata.getIndexInterval(), samplingLevel);
+                summaryBuilder = new IndexSummaryBuilder(estimatedKeys, metadata.getMinIndexInterval(), samplingLevel);
 
             long indexPosition;
             while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
@@ -664,7 +661,7 @@ public class SSTableReader extends SSTable implements Closeable
         try
         {
             iStream = new DataInputStream(new FileInputStream(summariesFile));
-            indexSummary = IndexSummary.serializer.deserialize(iStream, partitioner, descriptor.version.hasSamplingLevel, metadata.getIndexInterval());
+            indexSummary = IndexSummary.serializer.deserialize(iStream, partitioner, descriptor.version.hasSamplingLevel, metadata.getMinIndexInterval(), metadata.getMaxIndexInterval());
             first = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
             last = partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream));
             ibuilder.deserializeBounds(iStream);
@@ -738,10 +735,25 @@ public class SSTableReader extends SSTable implements Closeable
      */
     public SSTableReader cloneWithNewSummarySamplingLevel(int samplingLevel) throws IOException
     {
+        int minIndexInterval = metadata.getMinIndexInterval();
+        int maxIndexInterval = metadata.getMaxIndexInterval();
+        double effectiveInterval = indexSummary.getEffectiveIndexInterval();
+
         IndexSummary newSummary;
-        if (samplingLevel < indexSummary.getSamplingLevel())
+
+         // We have to rebuild the summary from the on-disk primary index in three cases:
+         // 1. The sampling level went up, so we need to read more entries off disk
+         // 2. The min_index_interval changed (in either direction); this changes what entries would be in the summary
+         //    at full sampling (and consequently at any other sampling level)
+         // 3. The max_index_interval was lowered, forcing us to raise the sampling level
+        if (samplingLevel > indexSummary.getSamplingLevel() || indexSummary.getMinIndexInterval() != minIndexInterval || effectiveInterval > maxIndexInterval)
         {
-            newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, partitioner);
+            newSummary = buildSummaryAtLevel(samplingLevel);
+        }
+        else if (samplingLevel < indexSummary.getSamplingLevel())
+        {
+            // we can use the existing index summary to make a smaller one
+            newSummary = IndexSummaryBuilder.downsample(indexSummary, samplingLevel, minIndexInterval, partitioner);
 
             SegmentedFile.Builder ibuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
             SegmentedFile.Builder dbuilder = compression
@@ -749,13 +761,10 @@ public class SSTableReader extends SSTable implements Closeable
                                            : SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
             saveSummary(ibuilder, dbuilder, newSummary);
         }
-        else if (samplingLevel > indexSummary.getSamplingLevel())
-        {
-            newSummary = upsampleSummary(samplingLevel);
-        }
         else
         {
-            throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level");
+            throw new AssertionError("Attempted to clone SSTableReader with the same index summary sampling level and " +
+                                     "no adjustments to min/max_index_interval");
         }
 
         markReplaced();
@@ -769,14 +778,14 @@ public class SSTableReader extends SSTable implements Closeable
         return replacement;
     }
 
-    private IndexSummary upsampleSummary(int newSamplingLevel) throws IOException
+    private IndexSummary buildSummaryAtLevel(int newSamplingLevel) throws IOException
     {
         // we read the positions in a BRAF so we don't have to worry about an entry spanning a mmap boundary.
         RandomAccessReader primaryIndex = RandomAccessReader.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)));
         try
         {
             long indexSize = primaryIndex.length();
-            IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getIndexInterval(), newSamplingLevel);
+            IndexSummaryBuilder summaryBuilder = new IndexSummaryBuilder(estimatedKeys(), metadata.getMinIndexInterval(), newSamplingLevel);
 
             long indexPosition;
             while ((indexPosition = primaryIndex.getFilePointer()) != indexSize)
@@ -803,6 +812,16 @@ public class SSTableReader extends SSTable implements Closeable
         return indexSummary.getOffHeapSize();
     }
 
+    public int getMinIndexInterval()
+    {
+        return indexSummary.getMinIndexInterval();
+    }
+
+    public double getEffectiveIndexInterval()
+    {
+        return indexSummary.getEffectiveIndexInterval();
+    }
+
     public void releaseSummary() throws IOException
     {
         indexSummary.close();
@@ -880,11 +899,11 @@ public class SSTableReader extends SSTable implements Closeable
     }
 
     /**
-     * @return An estimate of the number of keys in this SSTable.
+     * @return An estimate of the number of keys in this SSTable based on the index summary.
      */
     public long estimatedKeys()
     {
-        return ((long) indexSummary.getMaxNumberOfEntries()) * indexSummary.getIndexInterval();
+        return indexSummary.getEstimatedKeyCount();
     }
 
     /**
@@ -898,8 +917,8 @@ public class SSTableReader extends SSTable implements Closeable
         for (Pair<Integer, Integer> sampleIndexRange : sampleIndexes)
             sampleKeyCount += (sampleIndexRange.right - sampleIndexRange.left + 1);
 
-        // adjust for the current sampling level
-        long estimatedKeys = sampleKeyCount * (Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getIndexInterval()) / indexSummary.getSamplingLevel();
+        // adjust for the current sampling level: (BSL / SL) * index_interval_at_full_sampling
+        long estimatedKeys = sampleKeyCount * (Downsampling.BASE_SAMPLING_LEVEL * indexSummary.getMinIndexInterval()) / indexSummary.getSamplingLevel();
         return Math.max(1, estimatedKeys);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
index 9b50a18..ab1fe49 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
@@ -441,7 +441,7 @@ public class SSTableWriter extends SSTable
             indexFile = SequentialWriter.open(new File(descriptor.filenameFor(Component.PRIMARY_INDEX)),
                                               !metadata.populateIoCacheOnFlush());
             builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
-            summary = new IndexSummaryBuilder(keyCount, metadata.getIndexInterval(), Downsampling.BASE_SAMPLING_LEVEL);
+            summary = new IndexSummaryBuilder(keyCount, metadata.getMinIndexInterval(), Downsampling.BASE_SAMPLING_LEVEL);
             bf = FilterFactory.getFilter(keyCount, metadata.getBloomFilterFpChance(), true);
         }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index cca104e..91b4a36 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -2823,7 +2823,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         ColumnFamilyStore cfs = t.getColumnFamilyStore(cfName);
         List<DecoratedKey> keys = keySamples(Collections.singleton(cfs), range);
 
-        final long totalRowCountEstimate = (keys.size() + 1) * metadata.getIndexInterval();
+        long totalRowCountEstimate = cfs.estimatedKeysForRange(range);
 
         // splitCount should be much smaller than number of key samples, to avoid huge sampling error
         final int minSamplesPerSplit = 4;
@@ -2831,22 +2831,20 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         final int splitCount = Math.max(1, Math.min(maxSplitCount, (int)(totalRowCountEstimate / keysPerSplit)));
 
         List<Token> tokens = keysToTokens(range, keys);
-        return getSplits(tokens, splitCount, metadata);
+        return getSplits(tokens, splitCount, metadata, cfs);
     }
 
-    private List<Pair<Range<Token>, Long>> getSplits(List<Token> tokens, int splitCount, CFMetaData metadata)
+    private List<Pair<Range<Token>, Long>> getSplits(List<Token> tokens, int splitCount, CFMetaData metadata, ColumnFamilyStore cfs)
     {
         final double step = (double) (tokens.size() - 1) / splitCount;
-        int prevIndex = 0;
         Token prevToken = tokens.get(0);
         List<Pair<Range<Token>, Long>> splits = Lists.newArrayListWithExpectedSize(splitCount);
         for (int i = 1; i <= splitCount; i++)
         {
             int index = (int) Math.round(i * step);
             Token token = tokens.get(index);
-            long rowCountEstimate = (index - prevIndex) * metadata.getIndexInterval();
-            splits.add(Pair.create(new Range<Token>(prevToken, token), rowCountEstimate));
-            prevIndex = index;
+            Range<Token> range = new Range<>(prevToken, token);
+            splits.add(Pair.create(range, cfs.estimatedKeysForRange(range)));
             prevToken = token;
         }
         return splits;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
index ce23a47..d68ba10 100644
--- a/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
+++ b/test/long/org/apache/cassandra/db/compaction/LongCompactionsTest.java
@@ -117,7 +117,7 @@ public class LongCompactionsTest extends SchemaLoader
         cfs.clearUnsafe();
 
         final int ROWS_PER_SSTABLE = 10;
-        final int SSTABLES = cfs.metadata.getIndexInterval() * 3 / ROWS_PER_SSTABLE;
+        final int SSTABLES = cfs.metadata.getMinIndexInterval() * 3 / ROWS_PER_SSTABLE;
 
         // disable compaction while flushing
         cfs.disableAutoCompaction();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/unit/org/apache/cassandra/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 5efc266..fbac746 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -169,7 +169,8 @@ public class SchemaLoader
                                            standardCFMD(ks1, "legacyleveled")
                                                                                .compactionStrategyClass(LeveledCompactionStrategy.class)
                                                                                .compactionStrategyOptions(leveledOptions),
-                                           standardCFMD(ks1, "StandardLowIndexInterval").indexInterval(8).caching(CFMetaData.Caching.NONE)));
+                                           standardCFMD(ks1, "StandardLowIndexInterval").minIndexInterval(8).maxIndexInterval(256).caching(CFMetaData.Caching.NONE)));
+
 
         // Keyspace 2
         schema.add(KSMetaData.testMetadata(ks2,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
index de449d8..d462d96 100644
--- a/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/BlacklistingCompactionsTest.java
@@ -74,7 +74,7 @@ public class BlacklistingCompactionsTest extends SchemaLoader
         final ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("Standard1");
 
         final int ROWS_PER_SSTABLE = 10;
-        final int SSTABLES = cfs.metadata.getIndexInterval() * 2 / ROWS_PER_SSTABLE;
+        final int SSTABLES = cfs.metadata.getMinIndexInterval() * 2 / ROWS_PER_SSTABLE;
 
         cfs.setCompactionStrategyClass(compactionStrategy);
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
index 994a846..c0d9678 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryManagerTest.java
@@ -20,8 +20,12 @@ package org.apache.cassandra.io.sstable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import org.slf4j.Logger;
@@ -34,7 +38,6 @@ import org.apache.cassandra.db.filter.QueryFilter;
 import org.apache.cassandra.metrics.RestorableMeter;
 
 import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.io.sstable.Downsampling.MIN_SAMPLING_LEVEL;
 import static org.apache.cassandra.io.sstable.IndexSummaryManager.DOWNSAMPLE_THESHOLD;
 import static org.apache.cassandra.io.sstable.IndexSummaryManager.UPSAMPLE_THRESHOLD;
 import static org.apache.cassandra.io.sstable.IndexSummaryManager.redistributeSummaries;
@@ -47,6 +50,33 @@ public class IndexSummaryManagerTest extends SchemaLoader
 {
     private static final Logger logger = LoggerFactory.getLogger(IndexSummaryManagerTest.class);
 
+    int originalMinIndexInterval;
+    int originalMaxIndexInterval;
+    long originalCapacity;
+
+    @Before
+    public void beforeTest()
+    {
+        String ksname = "Keyspace1";
+        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
+        Keyspace keyspace = Keyspace.open(ksname);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+        originalMinIndexInterval = cfs.metadata.getMinIndexInterval();
+        originalMaxIndexInterval = cfs.metadata.getMaxIndexInterval();
+        originalCapacity = IndexSummaryManager.instance.getMemoryPoolCapacityInMB();
+    }
+
+    @After
+    public void afterTest()
+    {
+        String ksname = "Keyspace1";
+        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
+        Keyspace keyspace = Keyspace.open(ksname);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+        cfs.metadata.minIndexInterval(originalMinIndexInterval);
+        cfs.metadata.maxIndexInterval(originalMaxIndexInterval);
+        IndexSummaryManager.instance.setMemoryPoolCapacityInMB(originalCapacity);
+    }
 
     private static long totalOffHeapSize(List<SSTableReader> sstables)
     {
@@ -73,11 +103,11 @@ public class IndexSummaryManagerTest extends SchemaLoader
     {
         for (int i = 0; i < numRows; i++)
         {
-            DecoratedKey key = Util.dk(String.valueOf(i));
+            DecoratedKey key = Util.dk(String.format("%3d", i));
             QueryFilter filter = QueryFilter.getIdentityFilter(key, cfs.getColumnFamilyName(), System.currentTimeMillis());
             ColumnFamily row = cfs.getColumnFamily(filter);
             assertNotNull(row);
-            Cell cell = row.getColumn(Util.cellname("cell"));
+            Cell cell = row.getColumn(Util.cellname("column"));
             assertNotNull(cell);
             assertEquals(100, cell.value().array().length);
         }
@@ -91,37 +121,171 @@ public class IndexSummaryManagerTest extends SchemaLoader
         }
     };
 
-    @Test
-    public void testRedistributeSummaries() throws IOException
+    private void createSSTables(String ksname, String cfname, int numSSTables, int numRows)
     {
-        String ksname = "Keyspace1";
-        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
         Keyspace keyspace = Keyspace.open(ksname);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
         cfs.truncateBlocking();
         cfs.disableAutoCompaction();
 
+        ArrayList<Future> futures = new ArrayList<>(numSSTables);
         ByteBuffer value = ByteBuffer.wrap(new byte[100]);
-
-        int numSSTables = 4;
-        int numRows = 256;
         for (int sstable = 0; sstable < numSSTables; sstable++)
         {
             for (int row = 0; row < numRows; row++)
             {
-                DecoratedKey key = Util.dk(String.valueOf(row));
+                DecoratedKey key = Util.dk(String.format("%3d", row));
                 Mutation rm = new Mutation(ksname, key.key);
                 rm.add(cfname, Util.cellname("column"), value, 0);
-                rm.apply();
+                rm.applyUnsafe();
             }
-            cfs.forceBlockingFlush();
+            futures.add(cfs.forceFlush());
         }
+        for (Future future : futures)
+        {
+            try
+            {
+                future.get();
+            } catch (InterruptedException e)
+            {
+                throw new RuntimeException(e);
+            }
+            catch (ExecutionException e)
+            {
+                throw new RuntimeException(e);
+            }
+        }
+        assertEquals(numSSTables, cfs.getSSTables().size());
+        validateData(cfs, numRows);
+    }
+
+    @Test
+    public void testChangeMinIndexInterval() throws IOException
+    {
+        String ksname = "Keyspace1";
+        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
+        Keyspace keyspace = Keyspace.open(ksname);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+        int numSSTables = 1;
+        int numRows = 256;
+        createSSTables(ksname, cfname, numSSTables, numRows);
 
         List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
-        assertEquals(numSSTables, sstables.size());
-        validateData(cfs, numRows);
+        for (SSTableReader sstable : sstables)
+            sstable.readMeter = new RestorableMeter(100.0, 100.0);
 
         for (SSTableReader sstable : sstables)
+            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001);
+
+        // double the min_index_interval
+        cfs.metadata.minIndexInterval(originalMinIndexInterval * 2);
+        IndexSummaryManager.instance.redistributeSummaries();
+        for (SSTableReader sstable : cfs.getSSTables())
+        {
+            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001);
+            assertEquals(numRows / cfs.metadata.getMinIndexInterval(), sstable.getIndexSummarySize());
+        }
+
+        // return min_index_interval to its original value
+        cfs.metadata.minIndexInterval(originalMinIndexInterval);
+        IndexSummaryManager.instance.redistributeSummaries();
+        for (SSTableReader sstable : cfs.getSSTables())
+        {
+            assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001);
+            assertEquals(numRows / cfs.metadata.getMinIndexInterval(), sstable.getIndexSummarySize());
+        }
+
+        // halve the min_index_interval, but constrain the available space to exactly what we have now; as a result,
+        // the summary shouldn't change
+        cfs.metadata.minIndexInterval(originalMinIndexInterval / 2);
+        SSTableReader sstable = cfs.getSSTables().iterator().next();
+        long summarySpace = sstable.getIndexSummaryOffHeapSize();
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, Arrays.asList(sstable), summarySpace);
+        sstable = cfs.getSSTables().iterator().next();
+        assertEquals(originalMinIndexInterval, sstable.getEffectiveIndexInterval(), 0.001);
+        assertEquals(numRows / originalMinIndexInterval, sstable.getIndexSummarySize());
+
+        // keep the min_index_interval the same, but now give the summary enough space to grow by 50%
+        double previousInterval = sstable.getEffectiveIndexInterval();
+        int previousSize = sstable.getIndexSummarySize();
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, Arrays.asList(sstable), (long) Math.ceil(summarySpace * 1.5));
+        sstable = cfs.getSSTables().iterator().next();
+        assertEquals(previousSize * 1.5, (double) sstable.getIndexSummarySize(), 1);
+        assertEquals(previousInterval * (1.0 / 1.5), sstable.getEffectiveIndexInterval(), 0.001);
+
+        // return min_index_interval to it's original value (double it), but only give the summary enough space
+        // to have an effective index interval of twice the new min
+        cfs.metadata.minIndexInterval(originalMinIndexInterval);
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, Arrays.asList(sstable), (long) Math.ceil(summarySpace / 2));
+        sstable = cfs.getSSTables().iterator().next();
+        assertEquals(originalMinIndexInterval * 2, sstable.getEffectiveIndexInterval(), 0.001);
+        assertEquals(numRows / (originalMinIndexInterval * 2), sstable.getIndexSummarySize());
+
+        // raise the min_index_interval above our current effective interval, but set the max_index_interval lower
+        // than what we actually have space for (meaning the index summary would ideally be smaller, but this would
+        // result in an effective interval above the new max)
+        cfs.metadata.minIndexInterval(originalMinIndexInterval * 4);
+        cfs.metadata.maxIndexInterval(originalMinIndexInterval * 4);
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, Arrays.asList(sstable), 10);
+        sstable = cfs.getSSTables().iterator().next();
+        assertEquals(cfs.metadata.getMinIndexInterval(), sstable.getEffectiveIndexInterval(), 0.001);
+    }
+
+    @Test
+    public void testChangeMaxIndexInterval() throws IOException
+    {
+        String ksname = "Keyspace1";
+        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
+        Keyspace keyspace = Keyspace.open(ksname);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+        int numSSTables = 1;
+        int numRows = 256;
+        createSSTables(ksname, cfname, numSSTables, numRows);
+
+        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
+        for (SSTableReader sstable : sstables)
+            sstable.readMeter = new RestorableMeter(100.0, 100.0);
+
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, sstables, 1);
+        sstables = new ArrayList<>(cfs.getSSTables());
+        for (SSTableReader sstable : sstables)
+            assertEquals(cfs.metadata.getMaxIndexInterval(), sstable.getEffectiveIndexInterval(), 0.01);
+
+        // halve the max_index_interval
+        cfs.metadata.maxIndexInterval(cfs.metadata.getMaxIndexInterval() / 2);
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, sstables, 1);
+        sstables = new ArrayList<>(cfs.getSSTables());
+        for (SSTableReader sstable : sstables)
+        {
+            assertEquals(cfs.metadata.getMaxIndexInterval(), sstable.getEffectiveIndexInterval(), 0.01);
+            assertEquals(numRows / cfs.metadata.getMaxIndexInterval(), sstable.getIndexSummarySize());
+        }
+
+        // return max_index_interval to its original value
+        cfs.metadata.maxIndexInterval(cfs.metadata.getMaxIndexInterval() * 2);
+        IndexSummaryManager.redistributeSummaries(Collections.EMPTY_LIST, sstables, 1);
+        for (SSTableReader sstable : cfs.getSSTables())
+        {
+            assertEquals(cfs.metadata.getMaxIndexInterval(), sstable.getEffectiveIndexInterval(), 0.01);
+            assertEquals(numRows / cfs.metadata.getMaxIndexInterval(), sstable.getIndexSummarySize());
+        }
+    }
+
+    @Test(timeout = 10000)
+    public void testRedistributeSummaries() throws IOException
+    {
+        String ksname = "Keyspace1";
+        String cfname = "StandardLowIndexInterval"; // index interval of 8, no key caching
+        Keyspace keyspace = Keyspace.open(ksname);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfname);
+        int numSSTables = 4;
+        int numRows = 256;
+        createSSTables(ksname, cfname, numSSTables, numRows);
+
+        int minSamplingLevel = (BASE_SAMPLING_LEVEL * cfs.metadata.getMinIndexInterval()) / cfs.metadata.getMaxIndexInterval();
+
+        List<SSTableReader> sstables = new ArrayList<>(cfs.getSSTables());
+        for (SSTableReader sstable : sstables)
             sstable.readMeter = new RestorableMeter(100.0, 100.0);
 
         long singleSummaryOffHeapSpace = sstables.get(0).getIndexSummaryOffHeapSize();
@@ -147,7 +311,7 @@ public class IndexSummaryManagerTest extends SchemaLoader
         validateData(cfs, numRows);
 
         // upsample back up to half
-        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables,(singleSummaryOffHeapSpace * (numSSTables / 2)));
+        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables,(singleSummaryOffHeapSpace * (numSSTables / 2) + 4));
         assert sstables.size() == 4;
         for (SSTableReader sstable : sstables)
             assertEquals(BASE_SAMPLING_LEVEL / 2, sstable.getIndexSummarySamplingLevel());
@@ -194,7 +358,7 @@ public class IndexSummaryManagerTest extends SchemaLoader
         sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (singleSummaryOffHeapSpace * 3) + 50);
         Collections.sort(sstables, hotnessComparator);
 
-        if (sstables.get(0).getIndexSummarySamplingLevel() == MIN_SAMPLING_LEVEL)
+        if (sstables.get(0).getIndexSummarySamplingLevel() == minSamplingLevel)
             assertEquals(BASE_SAMPLING_LEVEL, sstables.get(1).getIndexSummarySamplingLevel());
         else
             assertEquals(BASE_SAMPLING_LEVEL, sstables.get(0).getIndexSummarySamplingLevel());
@@ -205,26 +369,26 @@ public class IndexSummaryManagerTest extends SchemaLoader
 
 
         // Cause a mix of upsampling and downsampling. We'll leave enough space for two full index summaries. The two
-        // coldest sstables will get downsampled to 8/128 of their size, leaving us with 1 and 112/128th index
+        // coldest sstables will get downsampled to 4/128 of their size, leaving us with 1 and 92/128th index
         // summaries worth of space.  The hottest sstable should get a full index summary, and the one in the middle
         // should get the remainder.
         sstables.get(0).readMeter = new RestorableMeter(0.0, 0.0);
         sstables.get(1).readMeter = new RestorableMeter(0.0, 0.0);
-        sstables.get(2).readMeter = new RestorableMeter(100, 100);
+        sstables.get(2).readMeter = new RestorableMeter(92, 92);
         sstables.get(3).readMeter = new RestorableMeter(128.0, 128.0);
-        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (100.0 / BASE_SAMPLING_LEVEL))));
+        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, (long) (singleSummaryOffHeapSpace + (singleSummaryOffHeapSpace * (92.0 / BASE_SAMPLING_LEVEL))));
         Collections.sort(sstables, hotnessComparator);
-        assertEquals(MIN_SAMPLING_LEVEL, sstables.get(0).getIndexSummarySamplingLevel());
-        assertEquals(MIN_SAMPLING_LEVEL, sstables.get(1).getIndexSummarySamplingLevel());
-        assertTrue(sstables.get(2).getIndexSummarySamplingLevel() > MIN_SAMPLING_LEVEL);
+        assertEquals(1, sstables.get(0).getIndexSummarySize());  // at the min sampling level
+        assertEquals(1, sstables.get(0).getIndexSummarySize());  // at the min sampling level
+        assertTrue(sstables.get(2).getIndexSummarySamplingLevel() > minSamplingLevel);
         assertTrue(sstables.get(2).getIndexSummarySamplingLevel() < BASE_SAMPLING_LEVEL);
         assertEquals(BASE_SAMPLING_LEVEL, sstables.get(3).getIndexSummarySamplingLevel());
         validateData(cfs, numRows);
 
         // Don't leave enough space for even the minimal index summaries
-        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, 100);
+        sstables = redistributeSummaries(Collections.EMPTY_LIST, sstables, 10);
         for (SSTableReader sstable : sstables)
-            assertEquals(MIN_SAMPLING_LEVEL, sstable.getIndexSummarySamplingLevel());
+            assertEquals(1, sstable.getIndexSummarySize());  // at the min sampling level
         validateData(cfs, numRows);
     }
 
@@ -255,11 +419,11 @@ public class IndexSummaryManagerTest extends SchemaLoader
         SSTableReader original = sstables.get(0);
 
         SSTableReader sstable = original;
-        for (int samplingLevel = MIN_SAMPLING_LEVEL; samplingLevel < BASE_SAMPLING_LEVEL; samplingLevel++)
+        for (int samplingLevel = 1; samplingLevel < BASE_SAMPLING_LEVEL; samplingLevel++)
         {
             sstable = sstable.cloneWithNewSummarySamplingLevel(samplingLevel);
             assertEquals(samplingLevel, sstable.getIndexSummarySamplingLevel());
-            int expectedSize = (numRows * samplingLevel) / (sstable.metadata.getIndexInterval() * BASE_SAMPLING_LEVEL);
+            int expectedSize = (numRows * samplingLevel) / (sstable.metadata.getMinIndexInterval() * BASE_SAMPLING_LEVEL);
             assertEquals(expectedSize, sstable.getIndexSummarySize(), 1);
         }
 
@@ -311,19 +475,20 @@ public class IndexSummaryManagerTest extends SchemaLoader
             cfs.forceBlockingFlush();
         }
 
-        assertEquals(1.0, manager.getAverageSamplingRatio(), 0.001);
-        Map<String, Double> samplingRatios = manager.getSamplingRatios();
-        for (Map.Entry<String, Double> entry : samplingRatios.entrySet())
-            assertEquals(1.0, entry.getValue(), 0.001);
+        assertTrue(manager.getAverageIndexInterval() >= cfs.metadata.getMinIndexInterval());
+        Map<String, Integer> intervals = manager.getIndexIntervals();
+        for (Map.Entry<String, Integer> entry : intervals.entrySet())
+            if (entry.getKey().contains("StandardLowIndexInterval"))
+                assertEquals(cfs.metadata.getMinIndexInterval(), entry.getValue(), 0.001);
 
         manager.setMemoryPoolCapacityInMB(0);
         manager.redistributeSummaries();
-        assertTrue(manager.getAverageSamplingRatio() < 0.99);
-        samplingRatios = manager.getSamplingRatios();
-        for (Map.Entry<String, Double> entry : samplingRatios.entrySet())
+        assertTrue(manager.getAverageIndexInterval() > cfs.metadata.getMinIndexInterval());
+        intervals = manager.getIndexIntervals();
+        for (Map.Entry<String, Integer> entry : intervals.entrySet())
         {
             if (entry.getKey().contains("StandardLowIndexInterval"))
-                assertTrue(entry.getValue() < 0.9);
+                assertTrue(entry.getValue() >= cfs.metadata.getMinIndexInterval());
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
index 2253302..ff991b4 100644
--- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java
@@ -39,7 +39,6 @@ import org.apache.cassandra.utils.Pair;
 import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.downsample;
 import static org.apache.cassandra.io.sstable.IndexSummaryBuilder.entriesAtSamplingLevel;
 import static org.apache.cassandra.io.sstable.Downsampling.BASE_SAMPLING_LEVEL;
-import static org.apache.cassandra.io.sstable.Downsampling.MIN_SAMPLING_LEVEL;
 
 import static org.junit.Assert.*;
 
@@ -81,7 +80,7 @@ public class IndexSummaryTest
         dos.writeUTF("JUNK");
         FileUtils.closeQuietly(dos);
         DataInputStream dis = new DataInputStream(new ByteArrayInputStream(aos.toByteArray()));
-        IndexSummary is = IndexSummary.serializer.deserialize(dis, DatabaseDescriptor.getPartitioner(), false, 1);
+        IndexSummary is = IndexSummary.serializer.deserialize(dis, DatabaseDescriptor.getPartitioner(), false, 1, 1);
         for (int i = 0; i < 100; i++)
             assertEquals(i, is.binarySearch(random.left.get(i)));
         // read the junk
@@ -105,7 +104,7 @@ public class IndexSummaryTest
         DataOutputStream dos = new DataOutputStream(aos);
         IndexSummary.serializer.serialize(summary, dos, false);
         DataInputStream dis = new DataInputStream(new ByteArrayInputStream(aos.toByteArray()));
-        IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, false, 1);
+        IndexSummary loaded = IndexSummary.serializer.deserialize(dis, p, false, 1, 1);
 
         assertEquals(1, loaded.size());
         assertEquals(summary.getPosition(0), loaded.getPosition(0));
@@ -135,10 +134,10 @@ public class IndexSummaryTest
         assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(0));
         assertEquals(Arrays.asList(0), Downsampling.getSamplingPattern(1));
 
-        assertEquals(Arrays.asList(0, 1), Downsampling.getSamplingPattern(2));
-        assertEquals(Arrays.asList(0, 2, 1, 3), Downsampling.getSamplingPattern(4));
-        assertEquals(Arrays.asList(0, 4, 2, 6, 1, 5, 3, 7), Downsampling.getSamplingPattern(8));
-        assertEquals(Arrays.asList(0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15), Downsampling.getSamplingPattern(16));
+        assertEquals(Arrays.asList(1, 0), Downsampling.getSamplingPattern(2));
+        assertEquals(Arrays.asList(3, 1, 2, 0), Downsampling.getSamplingPattern(4));
+        assertEquals(Arrays.asList(7, 3, 5, 1, 6, 2, 4, 0), Downsampling.getSamplingPattern(8));
+        assertEquals(Arrays.asList(15, 7, 11, 3, 13, 5, 9, 1, 14, 6, 10, 2, 12, 4, 8, 0), Downsampling.getSamplingPattern(16));
     }
 
     private static boolean shouldSkip(int index, List<Integer> startPoints)
@@ -171,9 +170,9 @@ public class IndexSummaryTest
 
         // downsample by one level, then two levels, then three levels...
         int downsamplingRound = 1;
-        for (int samplingLevel = BASE_SAMPLING_LEVEL - 1; samplingLevel >= MIN_SAMPLING_LEVEL; samplingLevel--)
+        for (int samplingLevel = BASE_SAMPLING_LEVEL - 1; samplingLevel >= 1; samplingLevel--)
         {
-            IndexSummary downsampled = downsample(original, samplingLevel, DatabaseDescriptor.getPartitioner());
+            IndexSummary downsampled = downsample(original, samplingLevel, 128, DatabaseDescriptor.getPartitioner());
             assertEquals(entriesAtSamplingLevel(samplingLevel, original.getMaxNumberOfEntries()), downsampled.size());
 
             int sampledCount = 0;
@@ -192,9 +191,9 @@ public class IndexSummaryTest
         // downsample one level each time
         IndexSummary previous = original;
         downsamplingRound = 1;
-        for (int downsampleLevel = BASE_SAMPLING_LEVEL - 1; downsampleLevel >= MIN_SAMPLING_LEVEL; downsampleLevel--)
+        for (int downsampleLevel = BASE_SAMPLING_LEVEL - 1; downsampleLevel >= 1; downsampleLevel--)
         {
-            IndexSummary downsampled = downsample(previous, downsampleLevel, DatabaseDescriptor.getPartitioner());
+            IndexSummary downsampled = downsample(previous, downsampleLevel, 128, DatabaseDescriptor.getPartitioner());
             assertEquals(entriesAtSamplingLevel(downsampleLevel, original.getMaxNumberOfEntries()), downsampled.size());
 
             int sampledCount = 0;
@@ -216,7 +215,7 @@ public class IndexSummaryTest
     @Test
     public void testOriginalIndexLookup()
     {
-        for (int i = BASE_SAMPLING_LEVEL; i >= MIN_SAMPLING_LEVEL; i--)
+        for (int i = BASE_SAMPLING_LEVEL; i >= 1; i--)
             assertEquals(i, Downsampling.getOriginalIndexes(i).size());
 
         ArrayList<Integer> full = new ArrayList<>();
@@ -224,13 +223,13 @@ public class IndexSummaryTest
             full.add(i);
 
         assertEquals(full, Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL));
-        // the entry at index 0 is the first to go
-        assertEquals(full.subList(1, full.size()), Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL - 1));
+        // the entry at index 127 is the first to go
+        assertEquals(full.subList(0, full.size() - 1), Downsampling.getOriginalIndexes(BASE_SAMPLING_LEVEL - 1));
 
         // spot check a few values (these depend on BASE_SAMPLING_LEVEL being 128)
         assert BASE_SAMPLING_LEVEL == 128;
-        assertEquals(Arrays.asList(31, 63, 95, 127), Downsampling.getOriginalIndexes(4));
-        assertEquals(Arrays.asList(63, 127), Downsampling.getOriginalIndexes(2));
+        assertEquals(Arrays.asList(0, 32, 64, 96), Downsampling.getOriginalIndexes(4));
+        assertEquals(Arrays.asList(0, 64), Downsampling.getOriginalIndexes(2));
         assertEquals(Arrays.asList(), Downsampling.getOriginalIndexes(0));
     }
 
@@ -241,13 +240,13 @@ public class IndexSummaryTest
         for (int i = 0; i < BASE_SAMPLING_LEVEL; i++)
             assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL, indexInterval));
 
-        // with one round of downsampling, only the first summary has been removed, so only the last index will have
+        // with one round of downsampling, only the last summary entry has been removed, so only the last index will have
         // double the gap until the next sample
         for (int i = 0; i < BASE_SAMPLING_LEVEL - 2; i++)
             assertEquals(indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(i, BASE_SAMPLING_LEVEL - 1, indexInterval));
         assertEquals(indexInterval * 2, Downsampling.getEffectiveIndexIntervalAfterIndex(BASE_SAMPLING_LEVEL - 2, BASE_SAMPLING_LEVEL - 1, indexInterval));
 
-        // at samplingLevel=2, the retained summary points are [63, 127] (assumes BASE_SAMPLING_LEVEL is 128)
+        // at samplingLevel=2, the retained summary points are [0, 64] (assumes BASE_SAMPLING_LEVEL is 128)
         assert BASE_SAMPLING_LEVEL == 128;
         assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(0, 2, indexInterval));
         assertEquals(64 * indexInterval, Downsampling.getEffectiveIndexIntervalAfterIndex(1, 2, indexInterval));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee477cc4/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 4b05e17..a7b8a3a 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -242,7 +242,7 @@ public class SSTableReaderTest extends SchemaLoader
 
         DecoratedKey firstKey = null, lastKey = null;
         long timestamp = System.currentTimeMillis();
-        for (int i = 0; i < store.metadata.getIndexInterval(); i++)
+        for (int i = 0; i < store.metadata.getMinIndexInterval(); i++)
         {
             DecoratedKey key = Util.dk(String.valueOf(i));
             if (firstKey == null)
@@ -359,7 +359,7 @@ public class SSTableReaderTest extends SchemaLoader
         final ColumnFamilyStore store = keyspace.getColumnFamilyStore("StandardLowIndexInterval"); // index interval of 8, no key caching
         CompactionManager.instance.disableAutoCompaction();
 
-        final int NUM_ROWS = 1000;
+        final int NUM_ROWS = 512;
         for (int j = 0; j < NUM_ROWS; j++)
         {
             ByteBuffer key = ByteBufferUtil.bytes(String.format("%3d", j));
@@ -402,7 +402,7 @@ public class SSTableReaderTest extends SchemaLoader
             }));
         }
 
-        SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(Downsampling.MIN_SAMPLING_LEVEL);
+        SSTableReader replacement = sstable.cloneWithNewSummarySamplingLevel(1);
         store.getDataTracker().replaceReaders(Arrays.asList(sstable), Arrays.asList(replacement));
         for (Future future : futures)
             future.get();


Mime
View raw message