cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yu...@apache.org
Subject cassandra git commit: ScrubTest should generate out-of-order SSTable on the fly
Date Thu, 30 Jul 2015 15:12:41 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk b090ed693 -> df3b6027b


ScrubTest should generate out-of-order SSTable on the fly

with some other ScrubTest fixes.

patch by yukim; reviewed by Stefania Alborghetti for CASSANDRA-9880


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/df3b6027
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/df3b6027
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/df3b6027

Branch: refs/heads/trunk
Commit: df3b6027bc90c658963e7bf72933b1507c6c6beb
Parents: b090ed6
Author: Yuki Morishita <yukim@apache.org>
Authored: Thu Jul 30 10:07:25 2015 -0500
Committer: Yuki Morishita <yukim@apache.org>
Committed: Thu Jul 30 10:09:07 2015 -0500

----------------------------------------------------------------------
 build.xml                                       |   4 -
 .../cassandra/db/compaction/Scrubber.java       |   7 +-
 .../io/sstable/format/SSTableReader.java        |   1 -
 .../cassandra/tools/StandaloneScrubber.java     |   2 +-
 test/data/corrupt-sstables/la-1-big-CRC.db      | Bin 8 -> 0 bytes
 test/data/corrupt-sstables/la-1-big-Data.db     | Bin 259 -> 0 bytes
 .../corrupt-sstables/la-1-big-Digest.adler32    |   1 -
 test/data/corrupt-sstables/la-1-big-Filter.db   | Bin 24 -> 0 bytes
 test/data/corrupt-sstables/la-1-big-Index.db    | Bin 105 -> 0 bytes
 .../corrupt-sstables/la-1-big-Statistics.db     | Bin 4645 -> 0 bytes
 test/data/corrupt-sstables/la-1-big-Summary.db  | Bin 83 -> 0 bytes
 test/data/corrupt-sstables/la-1-big-TOC.txt     |   8 -
 .../unit/org/apache/cassandra/db/ScrubTest.java | 180 ++++++++++---------
 13 files changed, 97 insertions(+), 106 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index b56e379..bc433ae 100644
--- a/build.xml
+++ b/build.xml
@@ -1281,7 +1281,6 @@
     <testmacro inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
       <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
       <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
-      <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
       <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
@@ -1302,7 +1301,6 @@
       <testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}" poffset="@{testlist.offset}"
exclude="**/*.java" timeout="${test.timeout}">
         <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
         <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
-        <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
         <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
         <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
         <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
@@ -1323,7 +1321,6 @@
                        exclude="**/*.java" timeout="${test.timeout}" testtag="compression">
         <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
         <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
-        <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
         <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
         <jvmarg value="-Dcassandra.test.compression=true"/>
         <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
@@ -1355,7 +1352,6 @@
       <test name="${test.name}" methods="${test.methods}"/>
       <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
       <jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
-      <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
       <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
       <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
       <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/db/compaction/Scrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
index c853157..81e307a 100644
--- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java
+++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java
@@ -59,7 +59,6 @@ public class Scrubber implements Closeable
     private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
 
     private final boolean isOffline;
-    private final boolean keepOriginals;
 
     private SSTableReader newSstable;
     private SSTableReader newInOrderSstable;
@@ -86,7 +85,7 @@ public class Scrubber implements Closeable
 
     public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted,
boolean isOffline, boolean checkData) throws IOException
     {
-        this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData,
false);
+        this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData);
     }
 
     @SuppressWarnings("resource")
@@ -95,8 +94,7 @@ public class Scrubber implements Closeable
                     boolean skipCorrupted,
                     OutputHandler outputHandler,
                     boolean isOffline,
-                    boolean checkData,
-                    boolean keepOriginals) throws IOException
+                    boolean checkData) throws IOException
     {
         this.cfs = cfs;
         this.transaction = transaction;
@@ -104,7 +102,6 @@ public class Scrubber implements Closeable
         this.outputHandler = outputHandler;
         this.skipCorrupted = skipCorrupted;
         this.isOffline = isOffline;
-        this.keepOriginals = keepOriginals;
         this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata,
                                                                                         
               sstable.descriptor.version,
                                                                                         
               sstable.header);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index e9ac200..6d39d2d 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -1251,7 +1251,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
     {
         if (this.first.compareTo(this.last) > 0)
         {
-            selfRef().release();
             throw new IllegalStateException(String.format("SSTable first key %s > last
key %s", this.first, this.last));
         }
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index c00d036..9388d98 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -121,7 +121,7 @@ public class StandaloneScrubber
                     try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB,
sstable))
                     {
                         txn.obsoleteOriginals(); // make sure originals are deleted and avoid
NPE if index is missing, CASSANDRA-9591
-                        try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted,
handler, true, !options.noValidate, false))
+                        try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted,
handler, true, !options.noValidate))
                         {
                             scrubber.scrub();
                         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-CRC.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-CRC.db b/test/data/corrupt-sstables/la-1-big-CRC.db
deleted file mode 100644
index 1a0c525..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-CRC.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Data.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Data.db b/test/data/corrupt-sstables/la-1-big-Data.db
deleted file mode 100644
index e6c5eb9..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Data.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Digest.adler32
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Digest.adler32 b/test/data/corrupt-sstables/la-1-big-Digest.adler32
deleted file mode 100644
index 93deb45..0000000
--- a/test/data/corrupt-sstables/la-1-big-Digest.adler32
+++ /dev/null
@@ -1 +0,0 @@
-3942663153
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Filter.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Filter.db b/test/data/corrupt-sstables/la-1-big-Filter.db
deleted file mode 100644
index 709d2ea..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Filter.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Index.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Index.db b/test/data/corrupt-sstables/la-1-big-Index.db
deleted file mode 100644
index 6e5e352..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Index.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Statistics.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Statistics.db b/test/data/corrupt-sstables/la-1-big-Statistics.db
deleted file mode 100644
index 15220e0..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Statistics.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Summary.db
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-Summary.db b/test/data/corrupt-sstables/la-1-big-Summary.db
deleted file mode 100644
index 732f27c..0000000
Binary files a/test/data/corrupt-sstables/la-1-big-Summary.db and /dev/null differ

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-TOC.txt
----------------------------------------------------------------------
diff --git a/test/data/corrupt-sstables/la-1-big-TOC.txt b/test/data/corrupt-sstables/la-1-big-TOC.txt
deleted file mode 100644
index 9ad71ef..0000000
--- a/test/data/corrupt-sstables/la-1-big-TOC.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-Statistics.db
-CRC.db
-TOC.txt
-Data.db
-Index.db
-Summary.db
-Digest.adler32
-Filter.db

http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index 9b62590..ee51a4d 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -18,54 +18,52 @@
  */
 package org.apache.cassandra.db;
 
+import java.io.File;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
 
-import java.io.File;
-import java.io.IOError;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-
-import org.apache.cassandra.Util;
-import org.apache.cassandra.UpdateBuilder;
-import org.apache.cassandra.cql3.QueryProcessor;
-import org.apache.cassandra.db.partitions.*;
-import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.exceptions.ConfigurationException;
-import org.apache.cassandra.exceptions.RequestExecutionException;
-import org.apache.cassandra.io.compress.CompressionMetadata;
-import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
-import org.apache.cassandra.config.*;
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.UpdateBuilder;
+import org.apache.cassandra.Util;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.compaction.Scrubber;
 import org.apache.cassandra.db.index.SecondaryIndex;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.marshal.*;
+import org.apache.cassandra.db.partitions.Partition;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.RequestExecutionException;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
+import org.apache.cassandra.io.compress.CompressionMetadata;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableRewriter;
-import org.apache.cassandra.OrderedJUnit4ClassRunner;
-import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.io.sstable.SSTableTxnWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.OutputHandler;
 
 import static org.junit.Assert.*;
 import static org.junit.Assume.assumeTrue;
@@ -306,74 +304,84 @@ public class ScrubTest
     @Test
     public void testScrubOutOfOrder() throws Exception
     {
-        CompactionManager.instance.disableAutoCompaction();
-        Keyspace keyspace = Keyspace.open(KEYSPACE);
-        String columnFamily = CF3;
-        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
-        cfs.clearUnsafe();
-
-        /*
-         * Code used to generate an outOfOrder sstable. The test for out-of-order key in
BigTableWriter must also be commented out.
-         * The test also assumes an ordered partitioner.
-        List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
-        SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))),
-        SSTableWriter writer = SSTableWriter.create(cfs.metadata,
-                                                    Descriptor.fromFilename(filename),
-                                                    keys.size(),
-                                                    0L,
-                                                    0,
-                                                    DatabaseDescriptor.getPartitioner(),
-                                                    SerializationHeader.make(cfs.metadata,
Collections.emptyList()));
-
-        for (String k : keys)
+        // This test assumes ByteOrderPartitioner to create out-of-order SSTable
+        IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner();
+        DatabaseDescriptor.setPartitioner(new ByteOrderedPartitioner());
+
+        // Create out-of-order SSTable
+        File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile();
+        // create ks/cf directory
+        File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3));
+        tempDataDir.mkdirs();
+        try
         {
-            RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0L, Util.dk(k));
-            PartitionUpdate update = builder.clustering("someName")
-                                            .add("val", "someValue")
-                                            .buildUpdate();
-
-            writer.append(update.unfilteredIterator());
-        }
-        writer.finish(false);
-         */
+            CompactionManager.instance.disableAutoCompaction();
+            Keyspace keyspace = Keyspace.open(KEYSPACE);
+            String columnFamily = CF3;
+            ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily);
+            cfs.clearUnsafe();
+
+            List<String> keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d");
+            String filename = cfs.getSSTablePath(tempDataDir);
+            Descriptor desc = Descriptor.fromFilename(filename);
+
+            try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc,
+                                                             keys.size(),
+                                                             0L,
+                                                             0,
+                                                             SerializationHeader.make(cfs.metadata,
+                                                                                      Collections.emptyList())))
+            {
 
-        String root = System.getProperty("corrupt-sstable-root");
-        assert root != null;
+                for (String k : keys)
+                {
+                    PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k))
+                                                          .newRow("someName").add("val",
"someValue")
+                                                          .build();
 
-        File rootDir = new File(root);
-        assert rootDir.isDirectory();
-        Descriptor desc = new Descriptor("la", rootDir, KEYSPACE, columnFamily, 1, SSTableFormat.Type.BIG);
-        CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname);
+                    writer.append(update.unfilteredIterator());
+                }
+                writer.finish(false);
+            }
 
-        try
-        {
-            SSTableReader.open(desc, metadata);
-            fail("SSTR validation should have caught the out-of-order rows");
+            try
+            {
+                SSTableReader.open(desc, cfs.metadata);
+                fail("SSTR validation should have caught the out-of-order rows");
+            }
+            catch (IllegalStateException ise)
+            { /* this is expected */ }
+
+            // open without validation for scrubbing
+            Set<Component> components = new HashSet<>();
+            if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+                components.add(Component.COMPRESSION_INFO);
+            components.add(Component.DATA);
+            components.add(Component.PRIMARY_INDEX);
+            components.add(Component.FILTER);
+            components.add(Component.STATS);
+            components.add(Component.SUMMARY);
+            components.add(Component.TOC);
+
+            SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
+            if (sstable.last.compareTo(sstable.first) < 0)
+                sstable.last = sstable.first;
+
+            try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB,
sstable);
+                 Scrubber scrubber = new Scrubber(cfs, txn, false, true, true))
+            {
+                scrubber.scrub();
+            }
+            cfs.loadNewSSTables();
+            assertOrderedAll(cfs, 7);
+            sstable.selfRef().release();
         }
-        catch (IllegalStateException ise) { /* this is expected */ }
-
-        // open without validation for scrubbing
-        Set<Component> components = new HashSet<>();
-        //components.add(Component.COMPRESSION_INFO);
-        components.add(Component.DATA);
-        components.add(Component.PRIMARY_INDEX);
-        components.add(Component.FILTER);
-        components.add(Component.STATS);
-        components.add(Component.SUMMARY);
-        components.add(Component.TOC);
-
-        SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs);
-        if (sstable.last.compareTo(sstable.first) < 0)
-            sstable.last = sstable.first;
-
-        try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB,
sstable);
-             Scrubber scrubber = new Scrubber(cfs, txn, false, new OutputHandler.LogOutput(),
true, true, true))
+        finally
         {
-            scrubber.scrub();
+            FileUtils.deleteRecursive(tempDataDir);
+            // reset partitioner
+            DatabaseDescriptor.setPartitioner(oldPartitioner);
         }
-        cfs.loadNewSSTables();
-        assertOrderedAll(cfs, 7);
-        sstable.selfRef().release();
     }
 
     private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2)
throws IOException


Mime
View raw message