cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject [09/15] git commit: resurrect ScrubTest
Date Wed, 24 Apr 2013 01:15:21 GMT
resurrect ScrubTest


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/3d557240
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/3d557240
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/3d557240

Branch: refs/heads/cassandra-1.2
Commit: 3d557240ae85f626852fe7eb14bf8cfb1077d1b2
Parents: 302267e
Author: Jonathan Ellis <jbellis@apache.org>
Authored: Fri Apr 19 13:43:26 2013 -0500
Committer: Jonathan Ellis <jbellis@apache.org>
Committed: Tue Apr 23 10:24:18 2013 -0500

----------------------------------------------------------------------
 .../corrupt-sstables/Keyspace1-Super5-f-2-Data.db  |  Bin 530200 -> 0 bytes
 .../Keyspace1-Super5-f-2-Filter.db                 |  Bin 496 -> 0 bytes
 .../corrupt-sstables/Keyspace1-Super5-f-2-Index.db |  Bin 1300 -> 0 bytes
 .../Keyspace1-Super5-f-2-Statistics.db             |  Bin 4264 -> 0 bytes
 test/unit/org/apache/cassandra/db/ScrubTest.java   |  211 +++++++++++++++
 5 files changed, 211 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db
deleted file mode 100644
index 0f95b07..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Data.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db
deleted file mode 100644
index 88aac99..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Filter.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db
deleted file mode 100644
index a7787c5..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Index.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db b/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db
deleted file mode 100644
index 6312c71..0000000
Binary files a/test/data/corrupt-sstables/Keyspace1-Super5-f-2-Statistics.db and /dev/null
differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/3d557240/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
new file mode 100644
index 0000000..26f0e78
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -0,0 +1,211 @@
+package org.apache.cassandra.db;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.Test;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.db.columniterator.IdentityQueryFilter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.CLibrary;
+
+import static org.apache.cassandra.Util.column;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class ScrubTest extends SchemaLoader
+{
+    public String TABLE = "Keyspace1";
+    public String CF = "Standard1";
+    public String CF2 = "Super5";
+    public String CF3 = "Standard2";
+
+    public String copySSTables(String cf) throws IOException
+    {
+        String root = System.getProperty("corrupt-sstable-root");
+        assert root != null;
+        File rootDir = new File(root);
+        assert rootDir.isDirectory();
+
+        File destDir = Directories.create(TABLE, cf).getDirectoryForNewSSTables(1);
+
+        String corruptSSTableName = null;
+
+        FileUtils.createDirectory(destDir);
+        for (File srcFile : rootDir.listFiles())
+        {
+            if (srcFile.getName().equals(".svn"))
+                continue;
+            if (!srcFile.getName().contains(cf))
+                continue;
+            File destFile = new File(destDir, srcFile.getName());
+            CLibrary.createHardLink(srcFile, destFile);
+
+            assert destFile.exists() : destFile.getAbsoluteFile();
+
+            if(destFile.getName().endsWith("Data.db"))
+                corruptSSTableName = destFile.getCanonicalPath();
+        }
+
+        assert corruptSSTableName != null;
+        return corruptSSTableName;
+    }
+
+    @Test
+    public void testScrubOneRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Table table = Table.open(TABLE);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(CF);
+
+        List<Row> rows;
+
+        // insert data and verify we get it back w/ range query
+        fillCF(cfs, 1);
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assertEquals(1, rows.size());
+
+        CompactionManager.instance.performScrub(cfs);
+
+        // check data is still there
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assertEquals(1, rows.size());
+    }
+
+    @Test
+    public void testScrubDeletedRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Table table = Table.open(TABLE);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(CF3);
+
+        RowMutation rm;
+        rm = new RowMutation(TABLE, ByteBufferUtil.bytes(1));
+        ColumnFamily cf = ColumnFamily.create(TABLE, CF3);
+        cf.delete(new DeletionInfo(0, 1)); // expired tombstone
+        rm.add(cf);
+        rm.applyUnsafe();
+        cfs.forceBlockingFlush();
+
+        CompactionManager.instance.performScrub(cfs);
+        assert cfs.getSSTables().isEmpty();
+    }
+
+    @Test
+    public void testScrubMultiRow() throws IOException, ExecutionException, InterruptedException,
ConfigurationException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Table table = Table.open(TABLE);
+        ColumnFamilyStore cfs = table.getColumnFamilyStore(CF);
+
+        List<Row> rows;
+
+        // insert data and verify we get it back w/ range query
+        fillCF(cfs, 10);
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assertEquals(10, rows.size());
+
+        CompactionManager.instance.performScrub(cfs);
+
+        // check data is still there
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assertEquals(10, rows.size());
+    }
+
+    @Test
+    public void testScubOutOfOrder() throws Exception
+    {
+         CompactionManager.instance.disableAutoCompaction();
+         Table table = Table.open(TABLE);
+         String columnFamily = "Standard3";
+         ColumnFamilyStore cfs = table.getColumnFamilyStore(columnFamily);
+
+        /*
+         * Code used to generate an outOfOrder sstable. The test for out-of-order key in
SSTableWriter must also be commented out.
+         * The test also assumes an ordered partitioner.
+         *
+         *  ColumnFamily cf = ColumnFamily.create(TABLE, columnFamily);
+         *  cf.addColumn(new Column(ByteBufferUtil.bytes("someName"), ByteBufferUtil.bytes("someValue"),
0L));
+
+         *  SSTableWriter writer = cfs.createCompactionWriter((long)DatabaseDescriptor.getIndexInterval(),
new File("."), Collections.<SSTableReader>emptyList());
+         *  writer.append(Util.dk("a"), cf);
+         *  writer.append(Util.dk("b"), cf);
+         *  writer.append(Util.dk("z"), cf);
+         *  writer.append(Util.dk("c"), cf);
+         *  writer.append(Util.dk("y"), cf);
+         *  writer.append(Util.dk("d"), cf);
+         *  writer.closeAndOpenReader();
+         */
+
+        copySSTables(columnFamily);
+        cfs.loadNewSSTables();
+        assert cfs.getSSTables().size() > 0;
+
+        List<Row> rows;
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assert !isRowOrdered(rows) : "'corrupt' test file actually was not";
+
+        CompactionManager.instance.performScrub(cfs);
+        rows = cfs.getRangeSlice(null, Util.range("", ""), 1000, new IdentityQueryFilter(),
null);
+        assert isRowOrdered(rows) : "Scrub failed: " + rows;
+        assert rows.size() == 6: "Got " + rows.size();
+    }
+
+    private static boolean isRowOrdered(List<Row> rows)
+    {
+        DecoratedKey prev = null;
+        for (Row row : rows)
+        {
+            if (prev != null && prev.compareTo(row.key) > 0)
+                return false;
+            prev = row.key;
+        }
+        return true;
+    }
+
+    protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable) throws ExecutionException,
InterruptedException, IOException
+    {
+        for (int i = 0; i < rowsPerSSTable; i++)
+        {
+            String key = String.valueOf(i);
+            // create a row and update the birthdate value, test that the index query fetches
the new version
+            RowMutation rm;
+            rm = new RowMutation(TABLE, ByteBufferUtil.bytes(key));
+            ColumnFamily cf = ColumnFamily.create(TABLE, CF);
+            cf.addColumn(column("c1", "1", 1L));
+            cf.addColumn(column("c2", "2", 1L));
+            rm.add(cf);
+            rm.applyUnsafe();
+        }
+
+        cfs.forceBlockingFlush();
+    }
+}


Mime
View raw message