cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject [15/23] cassandra git commit: Change commitlog and sstables to track dirty and clean intervals.
Date Fri, 05 Aug 2016 13:49:06 GMT
http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/src/java/org/apache/cassandra/utils/IntegerInterval.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/utils/IntegerInterval.java b/src/java/org/apache/cassandra/utils/IntegerInterval.java
new file mode 100644
index 0000000..03ad6e0
--- /dev/null
+++ b/src/java/org/apache/cassandra/utils/IntegerInterval.java
@@ -0,0 +1,227 @@
+package org.apache.cassandra.utils;
+
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Longs;
+
+/**
+ * Mutable integer interval class, thread-safe.
+ * Represents the interval [lower,upper].
+ */
+public class IntegerInterval
+{
+    volatile long interval;
+    private static AtomicLongFieldUpdater<IntegerInterval> intervalUpdater =
+            AtomicLongFieldUpdater.newUpdater(IntegerInterval.class, "interval");
+
+    private IntegerInterval(long interval)
+    {
+        this.interval = interval;
+    }
+
+    public IntegerInterval(int lower, int upper)
+    {
+        this(make(lower, upper));
+    }
+
+    public IntegerInterval(IntegerInterval src)
+    {
+        this(src.interval);
+    }
+
+    public int lower()
+    {
+        return lower(interval);
+    }
+
+    public int upper()
+    {
+        return upper(interval);
+    }
+
+    /**
+     * Expands the interval to cover the given value by extending one of its sides if necessary.
+     * Mutates this. Thread-safe.
+     */
+    public void expandToCover(int value)
+    {
+        long prev;
+        int lower;
+        int upper;
+        do
+        {
+            prev = interval;
+            upper = upper(prev);
+            lower = lower(prev);
+            if (value > upper) // common case
+                upper = value;
+            else if (value < lower)
+                lower = value;
+        }
+        while (!intervalUpdater.compareAndSet(this, prev, make(lower, upper)));
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Long.hashCode(interval);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (getClass() != obj.getClass())
+            return false;
+        IntegerInterval other = (IntegerInterval) obj;
+        return interval == other.interval;
+    }
+
+    public String toString()
+    {
+        long interval = this.interval;
+        return "[" + lower(interval) + "," + upper(interval) + "]";
+    }
+
+    private static long make(int lower, int upper)
+    {
+        assert lower <= upper;
+        return ((lower & 0xFFFFFFFFL) << 32) | upper & 0xFFFFFFFFL;
+    }
+
+    private static int lower(long interval)
+    {
+        return (int) (interval >>> 32);
+    }
+
+    private static int upper(long interval)
+    {
+        return (int) interval;
+    }
+
+
+    /**
+     * A mutable set of closed integer intervals, stored in normalized form (i.e. where overlapping intervals are
+     * converted to a single interval covering both). Thread-safe.
+     */
+    public static class Set
+    {
+        static long[] EMPTY = new long[0];
+
+        private volatile long[] ranges = EMPTY;
+
+        /**
+         * Adds an interval to the set, performing the necessary normalization.
+         */
+        public synchronized void add(int start, int end)
+        {
+            assert start <= end;
+            long[] ranges, newRanges;
+            {
+                ranges = this.ranges; // take local copy to avoid risk of it changing in the midst of operation
+
+                // extend ourselves to cover any ranges we overlap
+                // record directly preceding our end may extend past us, so take the max of our end and its
+                int rpos = Arrays.binarySearch(ranges, ((end & 0xFFFFFFFFL) << 32) | 0xFFFFFFFFL); // floor (i.e. greatest <=) of the end position
+                if (rpos < 0)
+                    rpos = (-1 - rpos) - 1;
+                if (rpos >= 0)
+                {
+                    int extend = upper(ranges[rpos]);
+                    if (extend > end)
+                        end = extend;
+                }
+    
+                // record directly preceding our start may extend into us; if it does, we take it as our start
+                int lpos = Arrays.binarySearch(ranges, ((start & 0xFFFFFFFFL) << 32) | 0); // lower (i.e. greatest <) of the start position
+                if (lpos < 0)
+                    lpos = -1 - lpos;
+                lpos -= 1;
+                if (lpos >= 0)
+                {
+                    if (upper(ranges[lpos]) >= start)
+                    {
+                        start = lower(ranges[lpos]);
+                        --lpos;
+                    }
+                }
+    
+                newRanges = new long[ranges.length - (rpos - lpos) + 1];
+                int dest = 0;
+                for (int i = 0; i <= lpos; ++i)
+                    newRanges[dest++] = ranges[i];
+                newRanges[dest++] = make(start, end);
+                for (int i = rpos + 1; i < ranges.length; ++i)
+                    newRanges[dest++] = ranges[i];
+            }
+            this.ranges = newRanges;
+        }
+
+        /**
+         * Returns true if the set completely covers the given interval.
+         */
+        public boolean covers(IntegerInterval iv)
+        {
+            long l = iv.interval;
+            return covers(lower(l), upper(l));
+        }
+
+        /**
+         * Returns true if the set completely covers the given interval.
+         */
+        public boolean covers(int start, int end)
+        {
+            long[] ranges = this.ranges; // take local copy to avoid risk of it changing in the midst of operation
+            int rpos = Arrays.binarySearch(ranges, ((start & 0xFFFFFFFFL) << 32) | 0xFFFFFFFFL);        // floor (i.e. greatest <=) of the end position
+            if (rpos < 0)
+                rpos = (-1 - rpos) - 1;
+            if (rpos == -1)
+                return false;
+            return upper(ranges[rpos]) >= end;
+        }
+
+        /**
+         * Returns a lower bound for the whole set. Will throw if set is not empty.
+         */
+        public int lowerBound()
+        {
+            return lower(ranges[0]);
+        }
+
+        /**
+         * Returns an upper bound for the whole set. Will throw if set is not empty.
+         */
+        public int upperBound()
+        {
+            long[] ranges = this.ranges; // take local copy to avoid risk of it changing in the midst of operation
+            return upper(ranges[ranges.length - 1]);
+        }
+
+        public Collection<IntegerInterval> intervals()
+        {
+            return Lists.transform(Longs.asList(ranges), iv -> new IntegerInterval(iv));
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Arrays.hashCode(ranges);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (getClass() != obj.getClass())
+                return false;
+            Set other = (Set) obj;
+            return Arrays.equals(ranges, other.ranges);
+        }
+
+        public String toString()
+        {
+            return "[" + intervals().stream().map(IntegerInterval::toString).collect(Collectors.joining(", ")) + "]";
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..2df95c9
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Data.db
new file mode 100644
index 0000000..c90b58d
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..76480b1
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+2048618157
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Index.db
new file mode 100644
index 0000000..3c716e9
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Statistics.db
new file mode 100644
index 0000000..43beef3
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..69c19f3
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Data.db
new file mode 100644
index 0000000..7027017
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..4b4078b
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+892998706
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Index.db
new file mode 100644
index 0000000..b2f5171
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Statistics.db
new file mode 100644
index 0000000..535d7df
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_compact/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..42876b0
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Data.db
new file mode 100644
index 0000000..8de00de
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..70a8c08
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+1609623183
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Index.db
new file mode 100644
index 0000000..690d2e4
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Statistics.db
new file mode 100644
index 0000000..d5bf83b
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..5ff1f27
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Data.db
new file mode 100644
index 0000000..7c47153
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..a6875fa
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+1205036423
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Index.db
new file mode 100644
index 0000000..b31055b
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Statistics.db
new file mode 100644
index 0000000..3463560
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_clust_counter_compact/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..0b7faea
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Data.db
new file mode 100644
index 0000000..5862341
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..ee0485a
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+34605693
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Index.db
new file mode 100644
index 0000000..b3094bf
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Statistics.db
new file mode 100644
index 0000000..124f9a8
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..adb7fc4
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Data.db
new file mode 100644
index 0000000..4a00428
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..36c7d92
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+4017973941
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Index.db
new file mode 100644
index 0000000..56f29df
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Statistics.db
new file mode 100644
index 0000000..ac35208
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_compact/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..0d9c077
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Data.db
new file mode 100644
index 0000000..8aadb48
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..7bb4450
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+1545836769
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Index.db
new file mode 100644
index 0000000..59e65ca
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Statistics.db
new file mode 100644
index 0000000..c707d9e
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-CompressionInfo.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..56c95a8
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-CompressionInfo.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Data.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Data.db
new file mode 100644
index 0000000..6a5f57f
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Digest.crc32 b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..45b9e94
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+4272819930
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Filter.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Filter.db
new file mode 100644
index 0000000..2e1d5d2
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Filter.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Index.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Index.db
new file mode 100644
index 0000000..d094f73
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Index.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Statistics.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Statistics.db
new file mode 100644
index 0000000..c3299a0
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Statistics.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Summary.db b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Summary.db
new file mode 100644
index 0000000..9b24e04
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-Summary.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-TOC.txt b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-TOC.txt
new file mode 100644
index 0000000..8ef7cb0
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_simple_counter_compact/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+TOC.txt
+Summary.db
+Filter.db
+Statistics.db
+Index.db
+Data.db
+CompressionInfo.db
+Digest.crc32

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
----------------------------------------------------------------------
diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
index d517055..02b26c7 100644
--- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
+++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java
@@ -246,7 +246,7 @@ public class CommitLogStressTest
             verifySizes(commitLog);
 
             commitLog.discardCompletedSegments(Schema.instance.getCFMetaData("Keyspace1", "Standard1").cfId,
-                                               discardedPos);
+                    ReplayPosition.NONE, discardedPos);
             threads.clear();
             System.out.format("Discarded at %s\n", discardedPos);
             verifySizes(commitLog);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/Util.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/Util.java b/test/unit/org/apache/cassandra/Util.java
index e7b1ffa..d04ca9b 100644
--- a/test/unit/org/apache/cassandra/Util.java
+++ b/test/unit/org/apache/cassandra/Util.java
@@ -19,7 +19,9 @@ package org.apache.cassandra;
  *
  */
 
+import java.io.Closeable;
 import java.io.EOFException;
+import java.io.IOError;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
@@ -39,6 +41,7 @@ import org.apache.cassandra.config.ColumnDefinition;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.Directories.DataDirectory;
 import org.apache.cassandra.db.compaction.AbstractCompactionTask;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -57,7 +60,6 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.CounterId;
 import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.concurrent.OpOrder;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -651,4 +653,21 @@ public class Util
             return queryStorage(cfs, orderGroup);
         }
     }
+
+    public static Closeable markDirectoriesUnwriteable(ColumnFamilyStore cfs)
+    {
+        try
+        {
+            for ( ; ; )
+            {
+                DataDirectory dir = cfs.getDirectories().getWriteableLocation(1);
+                BlacklistedDirectories.maybeMarkUnwritable(cfs.getDirectories().getLocationForDisk(dir));
+            }
+        }
+        catch (IOError e)
+        {
+            // Expected -- marked all directories as unwritable
+        }
+        return () -> BlacklistedDirectories.clearUnwritableUnsafe();
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/cql3/CQLTester.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index a213edf..e3dc220 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -385,9 +385,9 @@ public abstract class CQLTester
     {
         try
         {
-            String currentTable = currentTable();
-            if (currentTable != null)
-                Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable).forceMajorCompaction();
+            ColumnFamilyStore store = getCurrentColumnFamilyStore();
+            if (store != null)
+                store.forceMajorCompaction();
         }
         catch (InterruptedException | ExecutionException e)
         {
@@ -397,9 +397,9 @@ public abstract class CQLTester
 
     public void cleanupCache()
     {
-        String currentTable = currentTable();
-        if (currentTable != null)
-            Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable).cleanupCache();
+        ColumnFamilyStore store = getCurrentColumnFamilyStore();
+        if (store != null)
+            store.cleanupCache();
     }
 
     public static FunctionName parseFunctionName(String qualifiedName)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
index 1527b1e..26e7fe2 100644
--- a/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
+++ b/test/unit/org/apache/cassandra/cql3/OutOfSpaceTest.java
@@ -19,18 +19,16 @@ package org.apache.cassandra.cql3;
 
 import static junit.framework.Assert.fail;
 
-import java.io.IOError;
+import java.io.Closeable;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 
 import org.junit.Assert;
 import org.junit.Test;
 
+import org.apache.cassandra.Util;
 import org.apache.cassandra.config.Config.DiskFailurePolicy;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.BlacklistedDirectories;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories.DataDirectory;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogSegment;
 import org.apache.cassandra.db.Keyspace;
@@ -48,12 +46,11 @@ public class OutOfSpaceTest extends CQLTester
     public void testFlushUnwriteableDie() throws Throwable
     {
         makeTable();
-        markDirectoriesUnwriteable();
 
         KillerForTests killerForTests = new KillerForTests();
         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try
+        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.die);
             flushAndExpectError();
@@ -71,10 +68,9 @@ public class OutOfSpaceTest extends CQLTester
     public void testFlushUnwriteableStop() throws Throwable
     {
         makeTable();
-        markDirectoriesUnwriteable();
 
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try
+        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.stop);
             flushAndExpectError();
@@ -90,10 +86,9 @@ public class OutOfSpaceTest extends CQLTester
     public void testFlushUnwriteableIgnore() throws Throwable
     {
         makeTable();
-        markDirectoriesUnwriteable();
 
         DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
-        try
+        try (Closeable c = Util.markDirectoriesUnwriteable(getCurrentColumnFamilyStore()))
         {
             DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
             flushAndExpectError();
@@ -104,7 +99,6 @@ public class OutOfSpaceTest extends CQLTester
         }
 
         // Next flush should succeed.
-        makeTable();
         flush();
     }
 
@@ -117,23 +111,6 @@ public class OutOfSpaceTest extends CQLTester
             execute("INSERT INTO %s (a, b, c) VALUES ('key', 'column" + i + "', null);");
     }
 
-    public void markDirectoriesUnwriteable()
-    {
-        ColumnFamilyStore cfs = Keyspace.open(keyspace()).getColumnFamilyStore(currentTable());
-        try
-        {
-            for ( ; ; )
-            {
-                DataDirectory dir = cfs.getDirectories().getWriteableLocation(1);
-                BlacklistedDirectories.maybeMarkUnwritable(cfs.getDirectories().getLocationForDisk(dir));
-            }
-        }
-        catch (IOError e)
-        {
-            // Expected -- marked all directories as unwritable
-        }
-    }
-
     public void flushAndExpectError() throws InterruptedException, ExecutionException
     {
         try

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
index 39ba886..9a0ddb8 100644
--- a/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
+++ b/test/unit/org/apache/cassandra/db/commitlog/CommitLogTest.java
@@ -18,12 +18,7 @@
 */
 package org.apache.cassandra.db.commitlog;
 
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
+import java.io.*;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collection;
@@ -31,6 +26,7 @@ import java.util.Collections;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
+import java.util.function.BiConsumer;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
@@ -46,18 +42,18 @@ import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.ParameterizedClass;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.RowUpdateBuilder;
+import org.apache.cassandra.config.Config.DiskFailurePolicy;
+import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.CommitLogReplayer.CommitLogReplayException;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.compress.DeflateCompressor;
 import org.apache.cassandra.io.compress.LZ4Compressor;
 import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.utils.ByteBufferUtil;
@@ -235,7 +231,7 @@ public class CommitLogTest
         assert CommitLog.instance.activeSegments() == 2 : "Expecting 2 segments, got " + CommitLog.instance.activeSegments();
 
         UUID cfid2 = m2.getColumnFamilyIds().iterator().next();
-        CommitLog.instance.discardCompletedSegments(cfid2, CommitLog.instance.getContext());
+        CommitLog.instance.discardCompletedSegments(cfid2, ReplayPosition.NONE, CommitLog.instance.getContext());
 
         // Assert we still have both our segment
         assert CommitLog.instance.activeSegments() == 2 : "Expecting 2 segments, got " + CommitLog.instance.activeSegments();
@@ -262,7 +258,7 @@ public class CommitLogTest
         // "Flush": this won't delete anything
         UUID cfid1 = rm.getColumnFamilyIds().iterator().next();
         CommitLog.instance.sync(true);
-        CommitLog.instance.discardCompletedSegments(cfid1, CommitLog.instance.getContext());
+        CommitLog.instance.discardCompletedSegments(cfid1, ReplayPosition.NONE, CommitLog.instance.getContext());
 
         assert CommitLog.instance.activeSegments() == 1 : "Expecting 1 segment, got " + CommitLog.instance.activeSegments();
 
@@ -283,7 +279,7 @@ public class CommitLogTest
         // didn't write anything on cf1 since last flush (and we flush cf2)
 
         UUID cfid2 = rm2.getColumnFamilyIds().iterator().next();
-        CommitLog.instance.discardCompletedSegments(cfid2, CommitLog.instance.getContext());
+        CommitLog.instance.discardCompletedSegments(cfid2, ReplayPosition.NONE, CommitLog.instance.getContext());
 
         // Assert we still have both our segment
         assert CommitLog.instance.activeSegments() == 1 : "Expecting 1 segment, got " + CommitLog.instance.activeSegments();
@@ -494,8 +490,8 @@ public class CommitLogTest
             ReplayPosition position = CommitLog.instance.getContext();
             for (Keyspace ks : Keyspace.system())
                 for (ColumnFamilyStore syscfs : ks.getColumnFamilyStores())
-                    CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, position);
-            CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, position);
+                    CommitLog.instance.discardCompletedSegments(syscfs.metadata.cfId, ReplayPosition.NONE, position);
+            CommitLog.instance.discardCompletedSegments(cfs2.metadata.cfId, ReplayPosition.NONE, position);
             assertEquals(1, CommitLog.instance.activeSegments());
         }
         finally
@@ -532,5 +528,138 @@ public class CommitLogTest
             DatabaseDescriptor.setAutoSnapshot(originalState);
         }
     }
+
+    @Test
+    public void testUnwriteableFlushRecovery() throws ExecutionException, InterruptedException, IOException
+    {
+        CommitLog.instance.resetUnsafe(true);
+
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+        DiskFailurePolicy oldPolicy = DatabaseDescriptor.getDiskFailurePolicy();
+        try
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.ignore);
+
+            for (int i = 0 ; i < 5 ; i++)
+            {
+                new RowUpdateBuilder(cfs.metadata, 0, "k")
+                    .clustering("c" + i).add("val", ByteBuffer.allocate(100))
+                    .build()
+                    .apply();
+
+                if (i == 2)
+                {
+                    try (Closeable c = Util.markDirectoriesUnwriteable(cfs))
+                    {
+                        cfs.forceBlockingFlush();
+                    }
+                    catch (Throwable t)
+                    {
+                        // expected. Cause (after some wrappings) should be a write error
+                        while (!(t instanceof FSWriteError))
+                            t = t.getCause();
+                    }
+                }
+                else
+                    cfs.forceBlockingFlush();
+            }
+        }
+        finally
+        {
+            DatabaseDescriptor.setDiskFailurePolicy(oldPolicy);
+        }
+
+        CommitLog.instance.sync(true);
+        System.setProperty("cassandra.replayList", KEYSPACE1 + "." + STANDARD1);
+        // Currently we don't attempt to re-flush a memtable that failed, thus make sure data is replayed by commitlog.
+        // If retries work subsequent flushes should clear up error and this should change to expect 0.
+        Assert.assertEquals(1, CommitLog.instance.resetUnsafe(false));
+    }
+
+    public void testOutOfOrderFlushRecovery(BiConsumer<ColumnFamilyStore, Memtable> flushAction, boolean performCompaction)
+            throws ExecutionException, InterruptedException, IOException
+    {
+        CommitLog.instance.resetUnsafe(true);
+
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(STANDARD1);
+
+        for (int i = 0 ; i < 5 ; i++)
+        {
+            new RowUpdateBuilder(cfs.metadata, 0, "k")
+                .clustering("c" + i).add("val", ByteBuffer.allocate(100))
+                .build()
+                .apply();
+
+            Memtable current = cfs.getTracker().getView().getCurrentMemtable();
+            if (i == 2)
+                current.makeUnflushable();
+
+            flushAction.accept(cfs, current);
+        }
+        if (performCompaction)
+            cfs.forceMajorCompaction();
+        // Make sure metadata saves and reads fine
+        for (SSTableReader reader : cfs.getLiveSSTables())
+            reader.reloadSSTableMetadata();
+
+        CommitLog.instance.sync(true);
+        System.setProperty("cassandra.replayList", KEYSPACE1 + "." + STANDARD1);
+        // In the absence of error, this should be 0 because forceBlockingFlush/forceRecycleAllSegments would have
+        // persisted all data in the commit log. Because we know there was an error, there must be something left to
+        // replay.
+        Assert.assertEquals(1, CommitLog.instance.resetUnsafe(false));
+    }
+
+    BiConsumer<ColumnFamilyStore, Memtable> flush = (cfs, current) ->
+    {
+        try
+        {
+            cfs.forceBlockingFlush();
+        }
+        catch (Throwable t)
+        {
+            // expected after makeUnflushable. Cause (after some wrappings) should be a write error
+            while (!(t instanceof FSWriteError))
+                t = t.getCause();
+            // Wait for started flushes to complete.
+            cfs.switchMemtableIfCurrent(current);
+        }
+    };
+
+    BiConsumer<ColumnFamilyStore, Memtable> recycleSegments = (cfs, current) ->
+    {
+        // Move to new commit log segment and try to flush all data. Also delete segments that no longer contain
+        // flushed data.
+        // This does not stop on errors and should retain segments for which flushing failed.
+        CommitLog.instance.forceRecycleAllSegments();
+
+        // Wait for started flushes to complete.
+        cfs.switchMemtableIfCurrent(current);
+    };
+
+    @Test
+    public void testOutOfOrderFlushRecovery() throws ExecutionException, InterruptedException, IOException
+    {
+        testOutOfOrderFlushRecovery(flush, false);
+    }
+
+    @Test
+    public void testOutOfOrderLogDiscard() throws ExecutionException, InterruptedException, IOException
+    {
+        testOutOfOrderFlushRecovery(recycleSegments, false);
+    }
+
+    @Test
+    public void testOutOfOrderFlushRecoveryWithCompaction() throws ExecutionException, InterruptedException, IOException
+    {
+        testOutOfOrderFlushRecovery(flush, true);
+    }
+
+    @Test
+    public void testOutOfOrderLogDiscardWithCompaction() throws ExecutionException, InterruptedException, IOException
+    {
+        testOutOfOrderFlushRecovery(recycleSegments, true);
+    }
 }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
index 96e6ad6..cd1f295 100644
--- a/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/NeverPurgeTest.java
@@ -80,9 +80,9 @@ public class NeverPurgeTest extends CQLTester
         execute("DELETE FROM %s WHERE a=3");
         cfs.forceBlockingFlush();
         cfs.enableAutoCompaction();
-        while (cfs.getSSTables().size() > 1)
+        while (cfs.getLiveSSTables().size() > 1)
             Thread.sleep(100);
-        verifyContainsTombstones(cfs.getSSTables(), 3);
+        verifyContainsTombstones(cfs.getLiveSSTables(), 3);
     }
 
     private void testHelper(String deletionStatement) throws Throwable
@@ -94,7 +94,7 @@ public class NeverPurgeTest extends CQLTester
         Thread.sleep(1000);
         cfs.forceBlockingFlush();
         cfs.forceMajorCompaction();
-        verifyContainsTombstones(cfs.getSSTables(), 1);
+        verifyContainsTombstones(cfs.getLiveSSTables(), 1);
     }
 
     private void verifyContainsTombstones(Collection<SSTableReader> sstables, int expectedTombstoneCount) throws Exception

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index b8de711..479e4e2 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -300,9 +300,6 @@ public class TrackerTest
         SSTableReader reader = MockSchema.sstable(0, 10, false, cfs);
         tracker.replaceFlushed(prev2, Collections.singleton(reader));
         Assert.assertEquals(1, tracker.getView().sstables.size());
-        Assert.assertEquals(1, tracker.getView().premature.size());
-        tracker.permitCompactionOfFlushed(singleton(reader));
-        Assert.assertEquals(0, tracker.getView().premature.size());
         Assert.assertEquals(1, listener.received.size());
         Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
         listener.received.clear();
@@ -318,13 +315,14 @@ public class TrackerTest
         tracker.markFlushing(prev1);
         reader = MockSchema.sstable(0, 10, true, cfs);
         cfs.invalidate(false);
-        tracker.replaceFlushed(prev1, Collections.singleton(reader));
-        tracker.permitCompactionOfFlushed(Collections.singleton(reader));
+        tracker.replaceFlushed(prev1, singleton(reader));
         Assert.assertEquals(0, tracker.getView().sstables.size());
         Assert.assertEquals(0, tracker.getView().flushingMemtables.size());
         Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
-        Assert.assertEquals(reader, (((SSTableDeletingNotification) listener.received.get(0)).deleting));
-        Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(1)).removed.size());
+        Assert.assertEquals(3, listener.received.size());
+        Assert.assertEquals(singleton(reader), ((SSTableAddedNotification) listener.received.get(0)).added);
+        Assert.assertTrue(listener.received.get(1) instanceof SSTableDeletingNotification);
+        Assert.assertEquals(1, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
         DatabaseDescriptor.setIncrementalBackupsEnabled(backups);
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
index a5dceca..7e4ced1 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/ViewTest.java
@@ -212,6 +212,6 @@ public class ViewTest
         for (int i = 0 ; i < sstableCount ; i++)
             sstables.add(MockSchema.sstable(i, cfs));
         return new View(ImmutableList.copyOf(memtables), Collections.<Memtable>emptyList(), Helpers.identityMap(sstables),
-                        Collections.<SSTableReader, SSTableReader>emptyMap(), Collections.<SSTableReader>emptySet(), SSTableIntervalTree.build(sstables));
+                        Collections.<SSTableReader, SSTableReader>emptyMap(), SSTableIntervalTree.build(sstables));
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index 62228e3..cefee52 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -71,7 +71,7 @@ public class LegacySSTableTest
      * See {@link #testGenerateSstables()} to generate sstables.
      * Take care on commit as you need to add the sstable files using {@code git add -f}
      */
-    public static final String[] legacyVersions = {"mb", "ma", "la", "ka", "jb"};
+    public static final String[] legacyVersions = {"mc", "mb", "ma", "la", "ka", "jb"};
 
     // 1200 chars
     static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" +

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index c842b7f..a7e822f 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -781,7 +781,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase
 
         cfs.addSSTable(writeFile(cfs, 1000));
 
-        Collection<SSTableReader> allSSTables = cfs.getSSTables();
+        Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
         assertEquals(1, allSSTables.size());
         final Token firstToken = allSSTables.iterator().next().first.getToken();
         DatabaseDescriptor.setSSTablePreempiveOpenIntervalInMB(1);
@@ -880,7 +880,7 @@ public class SSTableRewriterTest extends SSTableWriterTestBase
         truncate(cfs);
 
         cfs.addSSTable(writeFile(cfs, 100));
-        Collection<SSTableReader> allSSTables = cfs.getSSTables();
+        Collection<SSTableReader> allSSTables = cfs.getLiveSSTables();
         assertEquals(1, allSSTables.size());
         final AtomicBoolean done = new AtomicBoolean(false);
         final AtomicBoolean failed = new AtomicBoolean(false);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index 93365ef..de12d57 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -32,6 +32,7 @@ import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.io.sstable.Component;
@@ -85,8 +86,7 @@ public class MetadataSerializerTest
 
         CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
         MetadataCollector collector = new MetadataCollector(cfm.comparator)
-                                          .commitLogLowerBound(cllb)
-                                          .commitLogUpperBound(club);
+                                          .commitLogIntervals(new IntervalSet(cllb, club));
 
         String partitioner = RandomPartitioner.class.getCanonicalName();
         double bfFpChance = 0.1;
@@ -106,6 +106,18 @@ public class MetadataSerializerTest
         testOldReadsNew("ma", "mb");
     }
 
+    @Test
+    public void testMaReadMc() throws IOException
+    {
+        testOldReadsNew("ma", "mc");
+    }
+
+    @Test
+    public void testMbReadMc() throws IOException
+    {
+        testOldReadsNew("mb", "mc");
+    }
+
     public void testOldReadsNew(String oldV, String newV) throws IOException
     {
         Map<MetadataType, MetadataComponent> originalMetadata = constructMetadata();

http://git-wip-us.apache.org/repos/asf/cassandra/blob/904cb5d1/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java b/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
new file mode 100644
index 0000000..44843fd
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/IntegerIntervalsTest.java
@@ -0,0 +1,326 @@
+package org.apache.cassandra.utils;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.IntegerInterval.Set;
+
+public class IntegerIntervalsTest
+{
+    int[] values = new int[] { Integer.MIN_VALUE, -2, -1, 0, 5, 9, 13, Integer.MAX_VALUE };
+
+    @Test
+    public void testMake()
+    {
+        IntegerInterval iv;
+        for (int i = 0; i < values.length; ++i)
+        {
+            for (int j = i; j < values.length; ++j)
+            {
+                iv = new IntegerInterval(values[i], values[j]);
+                assertEquals(values[i], iv.lower());
+                assertEquals(values[j], iv.upper());
+            }
+        }
+
+        for (int i = 0; i < values.length; ++i)
+        {
+            for (int j = 0; j < i; ++j)
+            {
+                try
+                {
+                    iv = new IntegerInterval(values[i], values[j]);
+                    fail("Assertion not thrown: " + values[i] + ", " + values[j]);
+                }
+                catch (AssertionError e)
+                {
+                    // expected
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testExpandToCoverSingleThread()
+    {
+        IntegerInterval iv;
+        for (int i = 0; i < values.length; ++i)
+        {
+            for (int j = i; j < values.length; ++j)
+            {
+                iv = new IntegerInterval(values[i], values[j]);
+                int k = 0;
+                for (; k < i; ++k)
+                {
+                    IntegerInterval v = new IntegerInterval(iv);
+                    v.expandToCover(values[k]);
+                    assertEquals(values[k], v.lower());
+                    assertEquals(values[j], v.upper());
+                }
+                for (; k < j; ++k)
+                {
+                    IntegerInterval v = new IntegerInterval(iv);
+                    v.expandToCover(values[k]);
+                    assertEquals(values[i], v.lower());
+                    assertEquals(values[j], v.upper());
+                }
+                for (; k < values.length; ++k)
+                {
+                    IntegerInterval v = new IntegerInterval(iv);
+                    v.expandToCover(values[k]);
+                    assertEquals(values[i], v.lower());
+                    assertEquals(values[k], v.upper());
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testExpandToCoverMultiThread() throws InterruptedException
+    {
+        Random r = new Random();
+        int threads = 16;
+        int streamSize = 1000000;
+        List<Callable<Void>> tasks = new ArrayList<>(threads);
+        final IntegerInterval interval = new IntegerInterval(0, 0);
+        int min = 0;
+        int max = 0;
+        for (int i = 0; i < threads; ++i)
+        {
+            final int seed = r.nextInt();
+            tasks.add(() -> {
+                new Random(seed).ints(streamSize).forEach(v -> interval.expandToCover(v));
+                return null;
+            });
+            min = Math.min(min, new Random(seed).ints(streamSize).min().getAsInt());
+            max = Math.max(max, new Random(seed).ints(streamSize).max().getAsInt());
+        }
+        for (Future<?> f : Executors.newFixedThreadPool(threads).invokeAll(tasks))
+            Futures.getUnchecked(f);
+        assertEquals(min, interval.lower());
+        assertEquals(max, interval.upper());
+    }
+
+    void testSetAdd(int l, int r, Integer... expected)
+    {
+        Set s = new Set();
+        s.add(-3, -1);
+        s.add(1, 3);
+        s.add(l, r);
+        assertArrayEquals(expected, s
+                                    .intervals()
+                                    .stream()
+                                    .flatMap(x -> ImmutableList.of(x.lower(), x.upper()).stream())
+                                    .toArray());
+    }
+
+    void testSetAdd(int l, int r, String expected)
+    {
+        Set s = new Set();
+        s.add(-3, -1);
+        s.add(1, 3);
+        s.add(l, r);
+        assertEquals(expected, s.toString());
+    }
+
+    @Test
+    public void testSetAdd()
+    {
+        testSetAdd(Integer.MIN_VALUE, -4, Integer.MIN_VALUE, -4, -3, -1, 1, 3);
+        testSetAdd(Integer.MIN_VALUE, -3, Integer.MIN_VALUE, -1, 1, 3);
+        testSetAdd(Integer.MIN_VALUE, -2, Integer.MIN_VALUE, -1, 1, 3);
+        testSetAdd(Integer.MIN_VALUE, -1, Integer.MIN_VALUE, -1, 1, 3);
+        testSetAdd(Integer.MIN_VALUE, 0, Integer.MIN_VALUE, 0, 1, 3);
+        testSetAdd(Integer.MIN_VALUE, 1, Integer.MIN_VALUE, 3);
+        testSetAdd(Integer.MIN_VALUE, 2, Integer.MIN_VALUE, 3);
+        testSetAdd(Integer.MIN_VALUE, 3, Integer.MIN_VALUE, 3);
+        testSetAdd(Integer.MIN_VALUE, Integer.MAX_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE);
+
+        testSetAdd(-5, -4, "[[-5,-4], [-3,-1], [1,3]]");
+        testSetAdd(-5, -3, -5, -1, 1, 3);
+        testSetAdd(-5, -2, -5, -1, 1, 3);
+        testSetAdd(-5, -1, -5, -1, 1, 3);
+        testSetAdd(-5, 0, -5, 0, 1, 3);
+        testSetAdd(-5, 1, -5, 3);
+        testSetAdd(-5, 2, -5, 3);
+        testSetAdd(-5, 3, -5, 3);
+        testSetAdd(-5, 4, -5, 4);
+        testSetAdd(-5, Integer.MAX_VALUE, -5, Integer.MAX_VALUE);
+
+        testSetAdd(-3, -3, -3, -1, 1, 3);
+        testSetAdd(-3, -2, -3, -1, 1, 3);
+        testSetAdd(-3, -1, -3, -1, 1, 3);
+        testSetAdd(-3, 0, -3, 0, 1, 3);
+        testSetAdd(-3, 1, "[[-3,3]]");
+        testSetAdd(-3, 2, -3, 3);
+        testSetAdd(-3, 3, -3, 3);
+        testSetAdd(-3, 4, -3, 4);
+        testSetAdd(-3, Integer.MAX_VALUE, -3, Integer.MAX_VALUE);
+
+        testSetAdd(-2, -2, -3, -1, 1, 3);
+        testSetAdd(-2, -1, -3, -1, 1, 3);
+        testSetAdd(-2, 0, "[[-3,0], [1,3]]");
+        testSetAdd(-2, 1, -3, 3);
+        testSetAdd(-2, 2, -3, 3);
+        testSetAdd(-2, 3, -3, 3);
+        testSetAdd(-2, 4, -3, 4);
+        testSetAdd(-2, Integer.MAX_VALUE, -3, Integer.MAX_VALUE);
+
+        testSetAdd(-1, -1, -3, -1, 1, 3);
+        testSetAdd(-1, 0, -3, 0, 1, 3);
+        testSetAdd(-1, 1, -3, 3);
+        testSetAdd(-1, 2, -3, 3);
+        testSetAdd(-1, 3, -3, 3);
+        testSetAdd(-1, 4, -3, 4);
+        testSetAdd(-1, Integer.MAX_VALUE, -3, Integer.MAX_VALUE);
+
+        testSetAdd(0, 0, -3, -1, 0, 0, 1, 3);
+        testSetAdd(0, 1, -3, -1, 0, 3);
+        testSetAdd(0, 2, -3, -1, 0, 3);
+        testSetAdd(0, 3, -3, -1, 0, 3);
+        testSetAdd(0, 4, -3, -1, 0, 4);
+        testSetAdd(0, Integer.MAX_VALUE, -3, -1, 0, Integer.MAX_VALUE);
+
+        testSetAdd(1, 1, -3, -1, 1, 3);
+        testSetAdd(1, 2, -3, -1, 1, 3);
+        testSetAdd(1, 3, -3, -1, 1, 3);
+        testSetAdd(1, 4, -3, -1, 1, 4);
+        testSetAdd(1, Integer.MAX_VALUE, -3, -1, 1, Integer.MAX_VALUE);
+
+        testSetAdd(2, 2, -3, -1, 1, 3);
+        testSetAdd(2, 3, -3, -1, 1, 3);
+        testSetAdd(2, 4, -3, -1, 1, 4);
+        testSetAdd(2, Integer.MAX_VALUE, -3, -1, 1, Integer.MAX_VALUE);
+
+        testSetAdd(3, 3, "[[-3,-1], [1,3]]");
+        testSetAdd(3, 4, -3, -1, 1, 4);
+        testSetAdd(3, Integer.MAX_VALUE, -3, -1, 1, Integer.MAX_VALUE);
+
+        testSetAdd(4, 5, -3, -1, 1, 3, 4, 5);
+        testSetAdd(4, Integer.MAX_VALUE, -3, -1, 1, 3, 4, Integer.MAX_VALUE);
+    }
+
+    @Test
+    public void testSetAddMultiThread() throws InterruptedException
+    {
+        Random r = new Random();
+        int threads = 16;
+        int streamSize = 10000;
+        List<Callable<Void>> tasks = new ArrayList<>(threads);
+        final IntegerInterval.Set st = new IntegerInterval.Set();
+        final IntegerInterval.Set mt = new IntegerInterval.Set();
+        for (int i = 0; i < threads; ++i)
+        {
+            final int seed = r.nextInt();
+            tasks.add(() -> {
+                new Random(seed)
+                    .ints(streamSize)
+                    .forEach(v -> mt.add(v, v + 5));
+                return null;
+            });
+            new Random(seed)
+                .ints(streamSize)
+                .forEach(v -> st.add(v, v + 5));
+        }
+        for (Future<?> f : Executors.newFixedThreadPool(threads).invokeAll(tasks))
+            Futures.getUnchecked(f);
+        assertEquals(st, mt);
+    }
+
+    void testSetCovers(int l, int r, boolean expected)
+    {
+        Set s = new Set();
+        s.add(-3, -1);
+        s.add(1, 3);
+        assertEquals(expected, s.covers(new IntegerInterval(l, r)));
+    }
+
+
+    @Test
+    public void testSetCovers()
+    {
+        testSetCovers(Integer.MIN_VALUE, -4, false);
+        testSetCovers(Integer.MIN_VALUE, -3, false);
+        testSetCovers(Integer.MIN_VALUE, -2, false);
+        testSetCovers(Integer.MIN_VALUE, -1, false);
+        testSetCovers(Integer.MIN_VALUE, 0, false);
+        testSetCovers(Integer.MIN_VALUE, 1, false);
+        testSetCovers(Integer.MIN_VALUE, 2, false);
+        testSetCovers(Integer.MIN_VALUE, 3, false);
+        testSetCovers(Integer.MIN_VALUE, Integer.MAX_VALUE, false);
+
+        testSetCovers(-5, -4, false);
+        testSetCovers(-5, -3, false);
+        testSetCovers(-5, -2, false);
+        testSetCovers(-5, -1, false);
+        testSetCovers(-5, 0, false);
+        testSetCovers(-5, 1, false);
+        testSetCovers(-5, 2, false);
+        testSetCovers(-5, 3, false);
+        testSetCovers(-5, 4, false);
+        testSetCovers(-5, Integer.MAX_VALUE, false);
+
+        testSetCovers(-3, -3, true);
+        testSetCovers(-3, -2, true);
+        testSetCovers(-3, -1, true);
+        testSetCovers(-3, 0, false);
+        testSetCovers(-3, 1, false);
+        testSetCovers(-3, 2, false);
+        testSetCovers(-3, 3, false);
+        testSetCovers(-3, 4, false);
+        testSetCovers(-3, Integer.MAX_VALUE, false);
+
+        testSetCovers(-2, -2, true);
+        testSetCovers(-2, -1, true);
+        testSetCovers(-2, 0, false);
+        testSetCovers(-2, 1, false);
+        testSetCovers(-2, 2, false);
+        testSetCovers(-2, 3, false);
+        testSetCovers(-2, 4, false);
+        testSetCovers(-2, Integer.MAX_VALUE, false);
+
+        testSetCovers(-1, -1, true);
+        testSetCovers(-1, 0, false);
+        testSetCovers(-1, 1, false);
+        testSetCovers(-1, 2, false);
+        testSetCovers(-1, 3, false);
+        testSetCovers(-1, 4, false);
+        testSetCovers(-1, Integer.MAX_VALUE, false);
+
+        testSetCovers(0, 0, false);
+        testSetCovers(0, 1, false);
+        testSetCovers(0, 2, false);
+        testSetCovers(0, 3, false);
+        testSetCovers(0, 4, false);
+        testSetCovers(0, Integer.MAX_VALUE, false);
+
+        testSetCovers(1, 1, true);
+        testSetCovers(1, 2, true);
+        testSetCovers(1, 3, true);
+        testSetCovers(1, 4, false);
+        testSetCovers(1, Integer.MAX_VALUE, false);
+
+        testSetCovers(2, 2, true);
+        testSetCovers(2, 3, true);
+        testSetCovers(2, 4, false);
+        testSetCovers(2, Integer.MAX_VALUE, false);
+
+        testSetCovers(3, 3, true);
+        testSetCovers(3, 4, false);
+        testSetCovers(3, Integer.MAX_VALUE, false);
+
+        testSetCovers(4, 5, false);
+        testSetCovers(4, Integer.MAX_VALUE, false);
+    }
+}


Mime
View raw message