cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bdeggles...@apache.org
Subject [2/6] cassandra git commit: Sstable min/max metadata can cause data loss
Date Tue, 06 Nov 2018 19:25:27 GMT
Sstable min/max metadata can cause data loss

Patch by Blake Eggleston; Reviewed by Sam Tunnicliffe for CASSANDRA-14861


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d60c7835
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d60c7835
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d60c7835

Branch: refs/heads/cassandra-3.11
Commit: d60c78358b6f599a83f3c112bfd6ce72c1129c9f
Parents: e4bac44
Author: Blake Eggleston <bdeggleston@gmail.com>
Authored: Wed Oct 31 15:55:48 2018 -0700
Committer: Blake Eggleston <bdeggleston@gmail.com>
Committed: Tue Nov 6 11:17:06 2018 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 src/java/org/apache/cassandra/db/Slice.java     |  25 +-----
 .../cassandra/io/sstable/format/Version.java    |   2 +
 .../io/sstable/format/big/BigFormat.java        |   9 ++
 .../io/sstable/metadata/MetadataCollector.java  |  26 ++----
 .../io/sstable/metadata/StatsMetadata.java      |  14 ++-
 .../mc-1-big-CompressionInfo.db                 | Bin 0 -> 43 bytes
 .../mc-1-big-Data.db                            | Bin 0 -> 65 bytes
 .../mc-1-big-Digest.crc32                       |   1 +
 .../mc-1-big-Filter.db                          | Bin 0 -> 16 bytes
 .../mc-1-big-Index.db                           | Bin 0 -> 8 bytes
 .../mc-1-big-Statistics.db                      | Bin 0 -> 4789 bytes
 .../mc-1-big-Summary.db                         | Bin 0 -> 56 bytes
 .../mc-1-big-TOC.txt                            |   8 ++
 .../db/SinglePartitionSliceCommandTest.java     |  87 +++++++++++++++++++
 .../cassandra/io/sstable/LegacySSTableTest.java |  35 +++++++-
 16 files changed, 165 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cc8e348..0fb1b86 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 3.0.18
+ * Sstable min/max metadata can cause data loss (CASSANDRA-14861)
  * Dropped columns can cause reverse sstable iteration to return prematurely (CASSANDRA-14838)
  * Legacy sstables with  multi block range tombstones create invalid bound sequences (CASSANDRA-14823)
  * Expand range tombstone validation checks to multiple interim request stages (CASSANDRA-14824)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/src/java/org/apache/cassandra/db/Slice.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java
index 3c645dc..f90c195 100644
--- a/src/java/org/apache/cassandra/db/Slice.java
+++ b/src/java/org/apache/cassandra/db/Slice.java
@@ -248,29 +248,8 @@ public class Slice
      */
     public boolean intersects(ClusteringComparator comparator, List<ByteBuffer> minClusteringValues,
List<ByteBuffer> maxClusteringValues)
     {
-        // If this slice start after max or end before min, it can't intersect
-        if (start.compareTo(comparator, maxClusteringValues) > 0 || end.compareTo(comparator,
minClusteringValues) < 0)
-            return false;
-
-        // We could safely return true here, but there's a minor optimization: if the first
component
-        // of the slice is restricted to a single value (typically the slice is [4:5, 4:7]),
we can
-        // check that the second component falls within the min/max for that component (and
repeat for
-        // all components).
-        for (int j = 0; j < minClusteringValues.size() && j < maxClusteringValues.size();
j++)
-        {
-            ByteBuffer s = j < start.size() ? start.get(j) : null;
-            ByteBuffer f = j < end.size() ? end.get(j) : null;
-
-            // we already know the first component falls within its min/max range (otherwise
we wouldn't get here)
-            if (j > 0 && (j < end.size() && comparator.compareComponent(j,
f, minClusteringValues.get(j)) < 0 ||
-                        j < start.size() && comparator.compareComponent(j, s,
maxClusteringValues.get(j)) > 0))
-                return false;
-
-            // if this component isn't equal in the start and finish, we don't need to check
any more
-            if (j >= start.size() || j >= end.size() || comparator.compareComponent(j,
s, f) != 0)
-                break;
-        }
-        return true;
+        // If this slice starts after max clustering or ends before min clustering, it can't
intersect
+        return start.compareTo(comparator, maxClusteringValues) <= 0 && end.compareTo(comparator,
minClusteringValues) >= 0;
     }
 
     public String toString(CFMetaData metadata)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index 96c5a6e..2b9dcbd 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -74,6 +74,8 @@ public abstract class Version
 
     public abstract boolean hasCommitLogIntervals();
 
+    public abstract boolean hasAccurateMinMax();
+
     public String getVersion()
     {
         return version;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 16f0beb..d4549dd 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -126,6 +126,7 @@ public class BigFormat implements SSTableFormat
         //             store rows natively
         // mb (3.0.7, 3.7): commit log lower bound included
         // mc (3.0.8, 3.9): commit log intervals included
+        // md (3.0.18, 3.11.4): corrected sstable min/max clustering
         //
         // NOTE: when adding a new version, please add that to LegacySSTableTest, too.
 
@@ -147,6 +148,7 @@ public class BigFormat implements SSTableFormat
         private final boolean hasOldBfHashOrder;
         private final boolean hasCommitLogLowerBound;
         private final boolean hasCommitLogIntervals;
+        private final boolean hasAccurateMinMax;
 
         /**
          * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
@@ -189,6 +191,7 @@ public class BigFormat implements SSTableFormat
             hasCommitLogLowerBound = (version.compareTo("lb") >= 0 && version.compareTo("ma")
< 0)
                                      || version.compareTo("mb") >= 0;
             hasCommitLogIntervals = version.compareTo("mc") >= 0;
+            hasAccurateMinMax = version.compareTo("md") >= 0;
         }
 
         @Override
@@ -264,6 +267,12 @@ public class BigFormat implements SSTableFormat
         }
 
         @Override
+        public boolean hasAccurateMinMax()
+        {
+            return hasAccurateMinMax;
+        }
+
+        @Override
         public boolean storeRows()
         {
             return storeRows;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 487a932..f48d0a6 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -17,15 +17,14 @@
  */
 package org.apache.cassandra.io.sstable.metadata;
 
-import java.io.File;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 
 import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
@@ -96,8 +95,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
     protected double compressionRatio = NO_COMPRESSION_RATIO;
     protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogramBuilder();
     protected int sstableLevel;
-    protected ByteBuffer[] minClusteringValues;
-    protected ByteBuffer[] maxClusteringValues;
+    private ClusteringPrefix minClustering = Slice.Bound.TOP;
+    private ClusteringPrefix maxClustering = Slice.Bound.BOTTOM;
     protected boolean hasLegacyCounterShards = false;
     protected long totalColumnsSet;
     protected long totalRows;
@@ -115,8 +114,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
     {
         this.comparator = comparator;
 
-        this.minClusteringValues = new ByteBuffer[comparator.size()];
-        this.maxClusteringValues = new ByteBuffer[comparator.size()];
     }
 
     public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator
comparator, int level)
@@ -231,14 +228,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
 
     public MetadataCollector updateClusteringValues(ClusteringPrefix clustering)
     {
-        int size = clustering.size();
-        for (int i = 0; i < size; i++)
-        {
-            AbstractType<?> type = comparator.subtype(i);
-            ByteBuffer newValue = clustering.get(i);
-            minClusteringValues[i] = maybeMinimize(min(minClusteringValues[i], newValue,
type));
-            maxClusteringValues[i] = maybeMinimize(max(maxClusteringValues[i], newValue,
type));
-        }
+        minClustering = comparator.compare(clustering, minClustering) < 0 ? clustering
: minClustering;
+        maxClustering = comparator.compare(clustering, maxClustering) > 0 ? clustering
: maxClustering;
         return this;
     }
 
@@ -280,6 +271,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
 
     public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner,
double bloomFilterFPChance, long repairedAt, SerializationHeader header)
     {
+        Preconditions.checkState(comparator.compare(maxClustering, minClustering) >= 0);
         Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
         components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
         components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
@@ -294,8 +286,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                                              compressionRatio,
                                                              estimatedTombstoneDropTime.build(),
                                                              sstableLevel,
-                                                             makeList(minClusteringValues),
-                                                             makeList(maxClusteringValues),
+                                                             makeList(minClustering.getRawValues()),
+                                                             makeList(maxClustering.getRawValues()),
                                                              hasLegacyCounterShards,
                                                              repairedAt,
                                                              totalColumnsSet,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
index 9971eaa..1994bca 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
@@ -319,15 +319,25 @@ public class StatsMetadata extends MetadataComponent
             if (version.hasRepairedAt())
                 repairedAt = in.readLong();
 
+            // for legacy sstables, we skip deserializing the min and max clustering value
+            // to prevent erroneously excluding sstables from reads (see CASSANDRA-14861)
             int colCount = in.readInt();
             List<ByteBuffer> minClusteringValues = new ArrayList<>(colCount);
             for (int i = 0; i < colCount; i++)
-                minClusteringValues.add(ByteBufferUtil.readWithShortLength(in));
+            {
+                ByteBuffer val = ByteBufferUtil.readWithShortLength(in);
+                if (version.hasAccurateMinMax())
+                    minClusteringValues.add(val);
+            }
 
             colCount = in.readInt();
             List<ByteBuffer> maxClusteringValues = new ArrayList<>(colCount);
             for (int i = 0; i < colCount; i++)
-                maxClusteringValues.add(ByteBufferUtil.readWithShortLength(in));
+            {
+                ByteBuffer val = ByteBufferUtil.readWithShortLength(in);
+                if (version.hasAccurateMinMax())
+                    maxClusteringValues.add(val);
+            }
 
             boolean hasLegacyCounterShards = true;
             if (version.tracksLegacyCounterShards())

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-CompressionInfo.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-CompressionInfo.db
new file mode 100644
index 0000000..df694ed
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-CompressionInfo.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Data.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Data.db
new file mode 100644
index 0000000..e3e3637
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Data.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Digest.crc32
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Digest.crc32
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Digest.crc32
new file mode 100644
index 0000000..394acb4
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Digest.crc32
@@ -0,0 +1 @@
+4091794686
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Filter.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Filter.db
new file mode 100644
index 0000000..b58e394
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Filter.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Index.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Index.db
new file mode 100644
index 0000000..e27f0f6
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Index.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Statistics.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Statistics.db
new file mode 100644
index 0000000..491277f
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Statistics.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Summary.db
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Summary.db
new file mode 100644
index 0000000..7756279
Binary files /dev/null and b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-Summary.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-TOC.txt
b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-TOC.txt
new file mode 100644
index 0000000..52b155b
--- /dev/null
+++ b/test/data/legacy-sstables/mc/legacy_tables/legacy_mc_inaccurate_min_max/mc-1-big-TOC.txt
@@ -0,0 +1,8 @@
+Digest.crc32
+CompressionInfo.db
+TOC.txt
+Summary.db
+Statistics.db
+Index.db
+Data.db
+Filter.db

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index b1a374f..2891687 100644
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@ -24,8 +24,14 @@ import static org.junit.Assert.*;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Iterables;
+import com.google.common.primitives.Ints;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -39,8 +45,10 @@ import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.config.ColumnDefinition;
 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.filter.AbstractClusteringIndexFilter;
 import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
 import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@ -50,6 +58,7 @@ import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.IntegerType;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.RangeTombstoneMarker;
@@ -62,6 +71,7 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.btree.BTreeSet;
@@ -378,4 +388,81 @@ public class SinglePartitionSliceCommandTest
         Assert.assertNotNull(ret);
         Assert.assertFalse(ret.isEmpty());
     }
+
+
+    public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q)
+    {
+        SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement;
+
+        List<Unfiltered> unfiltereds = new ArrayList<>();
+        SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT,
FBUtilities.nowInSeconds());
+        Assert.assertEquals(1, query.commands.size());
+        SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands);
+        try (ReadOrderGroup group = ReadOrderGroup.forCommand(command);
+             UnfilteredPartitionIterator partitions = command.executeLocally(group))
+        {
+            assert partitions.hasNext();
+            try (UnfilteredRowIterator partition = partitions.next())
+            {
+                while (partition.hasNext())
+                {
+                    Unfiltered next = partition.next();
+                    unfiltereds.add(next);
+                }
+            }
+            assert !partitions.hasNext();
+        }
+        return unfiltereds;
+    }
+
+    private static void assertQueryReturnsSingleRT(String query)
+    {
+        List<Unfiltered> unfiltereds = getUnfilteredsFromSinglePartition(query);
+        Assert.assertEquals(2, unfiltereds.size());
+        Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+        Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+        Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+        Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+    }
+
+    private static ByteBuffer bb(int v)
+    {
+        return Int32Type.instance.decompose(v);
+    }
+
+    /**
+     * tests the bug raised in CASSANDRA-14861, where the sstable min/max can
+     * exclude range tombstones for clustering ranges not also covered by rows
+     */
+    @Test
+    public void sstableFiltering()
+    {
+        QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max
(k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+        CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max");
+        ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+
+        QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max (k,
c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)");
+        QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=1");
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=1 AND c2=1");
+        cfs.forceBlockingFlush();
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=1 AND c2=1");
+
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=1 AND c2=1 AND c3=1"); // clustering names
+
+        cfs.truncateBlocking();
+
+        long nowMillis = System.currentTimeMillis();
+        Slice slice = Slice.make(new Clustering(bb(2), bb(3)), new Clustering(bb(10), bb(10)));
+        RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis),
+                                                                       Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis))));
+        PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(),
1);
+        update.add(rt);
+        new Mutation(update).apply();
+
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=3 AND c2=2");
+        cfs.forceBlockingFlush();
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=3 AND c2=2");
+        assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=3 AND c2=2 AND c3=2"); // clustering names
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d60c7835/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index d58ce8a..5c65b31 100644
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
+import com.google.common.collect.Iterables;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -35,11 +36,20 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.SelectStatement;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.ReadOrderGroup;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
 import org.apache.cassandra.db.compaction.Verifier;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -49,6 +59,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.big.BigFormat;
 import org.apache.cassandra.service.CacheService;
+import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.streaming.StreamPlan;
 import org.apache.cassandra.streaming.StreamSession;
@@ -257,6 +268,28 @@ public class LegacySSTableTest
         }
     }
 
+
+    @Test
+    public void testInaccurateSSTableMinMax() throws Exception
+    {
+        QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max
(k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+        loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", "");
+
+        /*
+         sstable has the following mutations:
+            INSERT INTO legacy_tables.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES
(100, 4, 4, 4, 4)
+            DELETE FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1<3
+         */
+
+        String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=1 AND c2=1";
+        List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query);
+        Assert.assertEquals(2, unfiltereds.size());
+        Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+        Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+        Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+        Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+    }
+
     @Test
     public void testVerifyOldSSTables() throws Exception
     {
@@ -524,7 +557,7 @@ public class LegacySSTableTest
         }
     }
 
-    private void copySstablesFromTestData(String table, File ksDir) throws IOException
+    public static void copySstablesFromTestData(String table, File ksDir) throws IOException
     {
         File cfDir = new File(ksDir, table);
         cfDir.mkdir();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message