cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [2/3] git commit: add back ScrubTest patch by Tyler Hobbs; reviewed by jbellis for CASSANDRA-5730
Date Thu, 26 Sep 2013 18:22:00 GMT
add back ScrubTest
patch by Tyler Hobbs; reviewed by jbellis for CASSANDRA-5730


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d4930307
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d4930307
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d4930307

Branch: refs/heads/trunk
Commit: d49303078459b0b2e0d40d9f79660ccdc0dc1fe0
Parents: b5c23cf
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Thu Sep 26 13:18:02 2013 -0500
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Thu Sep 26 13:19:04 2013 -0500

----------------------------------------------------------------------
 .../Keyspace1-Standard3-ja-1-CRC.db             | Bin 8 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-Data.db            | Bin 354 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-Digest.sha1        |   1 -
 .../Keyspace1-Standard3-ja-1-Filter.db          | Bin 176 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-Index.db           | Bin 90 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-Statistics.db      | Bin 4377 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-Summary.db         | Bin 83 -> 0 bytes
 .../Keyspace1-Standard3-ja-1-TOC.txt            |   8 -
 .../Keyspace1-Standard3-jb-1-CompressionInfo.db | Bin 0 -> 43 bytes
 .../Keyspace1-Standard3-jb-1-Data.db            | Bin 0 -> 133 bytes
 .../Keyspace1-Standard3-jb-1-Filter.db          | Bin 0 -> 24 bytes
 .../Keyspace1-Standard3-jb-1-Index.db           | Bin 0 -> 90 bytes
 .../Keyspace1-Standard3-jb-1-Statistics.db      | Bin 0 -> 4390 bytes
 .../Keyspace1-Standard3-jb-1-Summary.db         | Bin 0 -> 71 bytes
 .../Keyspace1-Standard3-jb-1-TOC.txt            |   7 +
 .../unit/org/apache/cassandra/db/ScrubTest.java | 206 +++++++++++++++++++
 16 files changed, 213 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db
deleted file mode 100644
index 3cc23b0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-CRC.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db
deleted file mode 100644
index 70e64e0..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Data.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1 b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
deleted file mode 100644
index c53d478..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Digest.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a9fbab0c12f097cfbf91a7b8731a20363daef547  Keyspace1-Standard3-ja-1-Data.db
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db
deleted file mode 100644
index df0734d..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Filter.db and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db
deleted file mode 100644
index 5da2914..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Index.db and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db
deleted file mode 100644
index 28e250b..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Statistics.db and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db
deleted file mode 100644
index 105e42c..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-Summary.db and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
deleted file mode 100644
index 39df2ce..0000000
--- a/test/data/corrupt-sstables/Keyspace1-Standard3-ja-1-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-Digest.sha1
-Filter.db
-Data.db
-Index.db
-Summary.db
-CRC.db
-TOC.txt

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
new file mode 100644
index 0000000..44d2e59
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-CompressionInfo.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db
new file mode 100644
index 0000000..f75c4e6
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Data.db differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
new file mode 100644
index 0000000..8f0a999
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Filter.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
new file mode 100644
index 0000000..da84fbc
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Index.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
new file mode 100644
index 0000000..0762615
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Statistics.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
new file mode 100644
index 0000000..376ca9d
Binary files /dev/null and b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-Summary.db
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
new file mode 100644
index 0000000..cf6efa8
--- /dev/null
+++ b/test/data/corrupt-sstables/Keyspace1-Standard3-jb-1-TOC.txt
@@ -0,0 +1,7 @@
+CompressionInfo.db
+TOC.txt
+Filter.db
+Statistics.db
+Data.db
+Summary.db
+Index.db

http://git-wip-us.apache.org/repos/asf/cassandra/blob/d4930307/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
new file mode 100644
index 0000000..f09aab3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -0,0 +1,206 @@
+package org.apache.cassandra.db;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.compaction.Scrubber;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.apache.cassandra.Util.column;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class ScrubTest extends SchemaLoader
+{
+    public String KEYSPACE = "Keyspace1";
+    public String CF = "Standard1";
+    public String CF3 = "Standard2";
+
+    @Test
+    public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+        List<Row> rows;
+
+        // insert data and verify we get it back w/ range query
+        fillCF(cfs, 1);
+        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+        assertEquals(1, rows.size());
+
+        CompactionManager.instance.performScrub(cfs);
+
+        // check data is still there
+        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+        assertEquals(1, rows.size());
+    }
+
+    @Test
+    public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF3);
+
+        ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF3);
+        cf.delete(new DeletionInfo(0, 1)); // expired tombstone
+        RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(1), cf);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        CompactionManager.instance.performScrub(cfs);
+        assert cfs.getSSTables().isEmpty();
+    }
+
+    @Test
+    public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
+
+        List<Row> rows;
+
+        // insert data and verify we get it back w/ range query
+        fillCF(cfs, 10);
+        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+        assertEquals(10, rows.size());
+
+        CompactionManager.instance.performScrub(cfs);
+
+        // check data is still there
+        rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
+        assertEquals(10, rows.size());
+    }
+
+    @Test
+    public void testScrubOutOfOrder() throws Exception
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        String columnFamily = "Standard3";
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+
+        /*
+         * Code used to generate an outOfOrder sstable. The test for out-of-order key in
SSTableWriter must also be commented out.
+         * The test also assumes an ordered partitioner.
+         *
+        ColumnFamily cf = ArrayBackedSortedColumns.factory.create(cfs.metadata);
+        cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"),
0L));
+
+        SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
+                                                 cfs.metadata.getIndexInterval(),
+                                                 cfs.metadata,
+                                                 cfs.partitioner,
+                                                 SSTableMetadata.createCollector(BytesType.instance));
+        writer.append(Util.dk("a"), cf);
+        writer.append(Util.dk("b"), cf);
+        writer.append(Util.dk("z"), cf);
+        writer.append(Util.dk("c"), cf);
+        writer.append(Util.dk("y"), cf);
+        writer.append(Util.dk("d"), cf);
+        writer.closeAndOpenReader();
+        */
+
+
+        String root = System.getProperty("corrupt-sstable-root");
+        assert root != null;
+        File rootDir = new File(root);
+        assert rootDir.isDirectory();
+        Descriptor desc = new Descriptor(new Descriptor.Version("jb"), rootDir, KEYSPACE,
columnFamily, 1, false);
+        CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
+
+        try
+        {
+            SSTableReader.open(desc, metadata);
+            fail("SSTR validation should have caught the out-of-order rows");
+        }
+        catch (IllegalStateException ise) { /* this is expected */ }
+
+        // open without validation for scrubbing
+        Set<Component> components = new HashSet<>();
+        components.add(Component.COMPRESSION_INFO);
+        components.add(Component.DATA);
+        components.add(Component.PRIMARY_INDEX);
+        components.add(Component.FILTER);
+        components.add(Component.STATS);
+        components.add(Component.SUMMARY);
+        components.add(Component.TOC);
+        SSTableReader sstable = SSTableReader.openNoValidation(desc, components, metadata);
+
+        Scrubber scrubber = new Scrubber(cfs, sstable);
+        scrubber.scrub();
+
+        cfs.loadNewSSTables();
+        List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(),
1000);
+        assert isRowOrdered(rows) : "Scrub failed: " + rows;
+        assert rows.size() == 6 : "Got " + rows.size();
+    }
+
+    private static boolean isRowOrdered(List<Row> rows)
+    {
+        DecoratedKey prev = null;
+        for (Row row : rows)
+        {
+            if (prev != null && prev.compareTo(row.key) > 0)
+                return false;
+            prev = row.key;
+        }
+        return true;
+    }
+
+    protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException,
InterruptedException, IOException
+    {
+        for (int i = 0; i < rowsPerSSTable; i++)
+        {
+            String key = String.valueOf(i);
+            // create a row and update the birthdate value, test that the index query fetches
the new version
+            ColumnFamily cf = TreeMapBackedSortedColumns.factory.create(KEYSPACE, CF);
+            cf.addColumn(column("c1", "1", 1L));
+            cf.addColumn(column("c2", "2", 1L));
+            RowMutation rm = new RowMutation(KEYSPACE, ByteBufferUtil.bytes(key), cf);
+            rm.applyUnsafe();
+        }
+
+        cfs.forceBlockingFlush();
+    }
+}


Mime
View raw message