Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D203B18C5C for ; Thu, 30 Jul 2015 15:13:03 +0000 (UTC) Received: (qmail 42888 invoked by uid 500); 30 Jul 2015 15:12:41 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 42853 invoked by uid 500); 30 Jul 2015 15:12:41 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 42842 invoked by uid 99); 30 Jul 2015 15:12:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Jul 2015 15:12:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8B201E7136; Thu, 30 Jul 2015 15:12:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yukim@apache.org To: commits@cassandra.apache.org Message-Id: <05f464310d5a4d109caf3ba1b7f21bdf@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: cassandra git commit: ScrubTest should generate out-of-order SSTable on the fly Date: Thu, 30 Jul 2015 15:12:41 +0000 (UTC) Repository: cassandra Updated Branches: refs/heads/trunk b090ed693 -> df3b6027b ScrubTest should generate out-of-order SSTable on the fly with some other ScrubTest fixes. patch by yukim; reviewed by Stefania Alborghetti for CASSANDRA-9880 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/df3b6027 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/df3b6027 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/df3b6027 Branch: refs/heads/trunk Commit: df3b6027bc90c658963e7bf72933b1507c6c6beb Parents: b090ed6 Author: Yuki Morishita Authored: Thu Jul 30 10:07:25 2015 -0500 Committer: Yuki Morishita Committed: Thu Jul 30 10:09:07 2015 -0500 ---------------------------------------------------------------------- build.xml | 4 - .../cassandra/db/compaction/Scrubber.java | 7 +- .../io/sstable/format/SSTableReader.java | 1 - .../cassandra/tools/StandaloneScrubber.java | 2 +- test/data/corrupt-sstables/la-1-big-CRC.db | Bin 8 -> 0 bytes test/data/corrupt-sstables/la-1-big-Data.db | Bin 259 -> 0 bytes .../corrupt-sstables/la-1-big-Digest.adler32 | 1 - test/data/corrupt-sstables/la-1-big-Filter.db | Bin 24 -> 0 bytes test/data/corrupt-sstables/la-1-big-Index.db | Bin 105 -> 0 bytes .../corrupt-sstables/la-1-big-Statistics.db | Bin 4645 -> 0 bytes test/data/corrupt-sstables/la-1-big-Summary.db | Bin 83 -> 0 bytes test/data/corrupt-sstables/la-1-big-TOC.txt | 8 - .../unit/org/apache/cassandra/db/ScrubTest.java | 180 ++++++++++--------- 13 files changed, 97 insertions(+), 106 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index b56e379..bc433ae 100644 --- a/build.xml +++ b/build.xml @@ -1281,7 +1281,6 @@ - @@ -1302,7 +1301,6 @@ - @@ -1323,7 +1321,6 @@ exclude="**/*.java" timeout="${test.timeout}" testtag="compression"> - @@ -1355,7 +1352,6 @@ - http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/db/compaction/Scrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index c853157..81e307a 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -59,7 +59,6 @@ public class Scrubber implements Closeable private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer; private final boolean isOffline; - private final boolean keepOriginals; private SSTableReader newSstable; private SSTableReader newInOrderSstable; @@ -86,7 +85,7 @@ public class Scrubber implements Closeable public Scrubber(ColumnFamilyStore cfs, LifecycleTransaction transaction, boolean skipCorrupted, boolean isOffline, boolean checkData) throws IOException { - this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData, false); + this(cfs, transaction, skipCorrupted, new OutputHandler.LogOutput(), isOffline, checkData); } @SuppressWarnings("resource") @@ -95,8 +94,7 @@ public class Scrubber implements Closeable boolean skipCorrupted, OutputHandler outputHandler, boolean isOffline, - boolean checkData, - boolean keepOriginals) throws IOException + boolean checkData) throws IOException { this.cfs = cfs; this.transaction = transaction; @@ -104,7 +102,6 @@ public class Scrubber implements Closeable this.outputHandler = outputHandler; this.skipCorrupted = skipCorrupted; this.isOffline = isOffline; - this.keepOriginals = keepOriginals; this.rowIndexEntrySerializer = sstable.descriptor.version.getSSTableFormat().getIndexSerializer(sstable.metadata, sstable.descriptor.version, sstable.header); http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index e9ac200..6d39d2d 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -1251,7 +1251,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted 0) { - selfRef().release(); throw new IllegalStateException(String.format("SSTable first key %s > last key %s", this.first, this.last)); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/src/java/org/apache/cassandra/tools/StandaloneScrubber.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java index c00d036..9388d98 100644 --- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java +++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java @@ -121,7 +121,7 @@ public class StandaloneScrubber try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable)) { txn.obsoleteOriginals(); // make sure originals are deleted and avoid NPE if index is missing, CASSANDRA-9591 - try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate, false)) + try (Scrubber scrubber = new Scrubber(cfs, txn, options.skipCorrupted, handler, true, !options.noValidate)) { scrubber.scrub(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-CRC.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-CRC.db b/test/data/corrupt-sstables/la-1-big-CRC.db deleted file mode 100644 index 1a0c525..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-CRC.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Data.db b/test/data/corrupt-sstables/la-1-big-Data.db deleted file mode 100644 index e6c5eb9..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-Data.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Digest.adler32 ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Digest.adler32 b/test/data/corrupt-sstables/la-1-big-Digest.adler32 deleted file mode 100644 index 93deb45..0000000 --- a/test/data/corrupt-sstables/la-1-big-Digest.adler32 +++ /dev/null @@ -1 +0,0 @@ -3942663153 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Filter.db b/test/data/corrupt-sstables/la-1-big-Filter.db deleted file mode 100644 index 709d2ea..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-Filter.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Index.db b/test/data/corrupt-sstables/la-1-big-Index.db deleted file mode 100644 index 6e5e352..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-Index.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Statistics.db b/test/data/corrupt-sstables/la-1-big-Statistics.db deleted file mode 100644 index 15220e0..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-Statistics.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-Summary.db ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-Summary.db b/test/data/corrupt-sstables/la-1-big-Summary.db deleted file mode 100644 index 732f27c..0000000 Binary files a/test/data/corrupt-sstables/la-1-big-Summary.db and /dev/null differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/data/corrupt-sstables/la-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/corrupt-sstables/la-1-big-TOC.txt b/test/data/corrupt-sstables/la-1-big-TOC.txt deleted file mode 100644 index 9ad71ef..0000000 --- a/test/data/corrupt-sstables/la-1-big-TOC.txt +++ /dev/null @@ -1,8 +0,0 @@ -Statistics.db -CRC.db -TOC.txt -Data.db -Index.db -Summary.db -Digest.adler32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/df3b6027/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java index 9b62590..ee51a4d 100644 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@ -18,54 +18,52 @@ */ package org.apache.cassandra.db; +import java.io.File; +import java.io.IOError; +import java.io.IOException; +import java.io.RandomAccessFile; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ExecutionException; -import java.io.File; -import java.io.IOError; -import java.io.IOException; -import java.io.RandomAccessFile; - -import org.apache.cassandra.Util; -import org.apache.cassandra.UpdateBuilder; -import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.db.partitions.*; -import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.marshal.UUIDType; -import org.apache.cassandra.db.compaction.OperationType; -import org.apache.cassandra.db.lifecycle.LifecycleTransaction; -import org.apache.cassandra.db.rows.Row; -import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.io.compress.CompressionMetadata; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.SSTableWriter; -import org.apache.cassandra.schema.KeyspaceParams; import org.apache.commons.lang3.StringUtils; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; -import org.apache.cassandra.config.*; +import org.apache.cassandra.OrderedJUnit4ClassRunner; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.UpdateBuilder; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Operator; +import org.apache.cassandra.cql3.QueryProcessor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.compaction.Scrubber; import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.RequestExecutionException; import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.io.compress.CompressionMetadata; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.SSTableRewriter; -import org.apache.cassandra.OrderedJUnit4ClassRunner; -import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.io.sstable.SSTableTxnWriter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.OutputHandler; import static org.junit.Assert.*; import static org.junit.Assume.assumeTrue; @@ -306,74 +304,84 @@ public class ScrubTest @Test public void testScrubOutOfOrder() throws Exception { - CompactionManager.instance.disableAutoCompaction(); - Keyspace keyspace = Keyspace.open(KEYSPACE); - String columnFamily = CF3; - ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily); - cfs.clearUnsafe(); - - /* - * Code used to generate an outOfOrder sstable. The test for out-of-order key in BigTableWriter must also be commented out. - * The test also assumes an ordered partitioner. - List keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d"); - SSTableWriter writer = new SSTableWriter(cfs.getTempSSTablePath(new File(System.getProperty("corrupt-sstable-root"))), - SSTableWriter writer = SSTableWriter.create(cfs.metadata, - Descriptor.fromFilename(filename), - keys.size(), - 0L, - 0, - DatabaseDescriptor.getPartitioner(), - SerializationHeader.make(cfs.metadata, Collections.emptyList())); - - for (String k : keys) + // This test assumes ByteOrderPartitioner to create out-of-order SSTable + IPartitioner oldPartitioner = DatabaseDescriptor.getPartitioner(); + DatabaseDescriptor.setPartitioner(new ByteOrderedPartitioner()); + + // Create out-of-order SSTable + File tempDir = File.createTempFile("ScrubTest.testScrubOutOfOrder", "").getParentFile(); + // create ks/cf directory + File tempDataDir = new File(tempDir, String.join(File.separator, KEYSPACE, CF3)); + tempDataDir.mkdirs(); + try { - RowUpdateBuilder builder = new RowUpdateBuilder(cfs.metadata, 0L, Util.dk(k)); - PartitionUpdate update = builder.clustering("someName") - .add("val", "someValue") - .buildUpdate(); - - writer.append(update.unfilteredIterator()); - } - writer.finish(false); - */ + CompactionManager.instance.disableAutoCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE); + String columnFamily = CF3; + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnFamily); + cfs.clearUnsafe(); + + List keys = Arrays.asList("t", "a", "b", "z", "c", "y", "d"); + String filename = cfs.getSSTablePath(tempDataDir); + Descriptor desc = Descriptor.fromFilename(filename); + + try (SSTableTxnWriter writer = SSTableTxnWriter.create(desc, + keys.size(), + 0L, + 0, + SerializationHeader.make(cfs.metadata, + Collections.emptyList()))) + { - String root = System.getProperty("corrupt-sstable-root"); - assert root != null; + for (String k : keys) + { + PartitionUpdate update = UpdateBuilder.create(cfs.metadata, Util.dk(k)) + .newRow("someName").add("val", "someValue") + .build(); - File rootDir = new File(root); - assert rootDir.isDirectory(); - Descriptor desc = new Descriptor("la", rootDir, KEYSPACE, columnFamily, 1, SSTableFormat.Type.BIG); - CFMetaData metadata = Schema.instance.getCFMetaData(desc.ksname, desc.cfname); + writer.append(update.unfilteredIterator()); + } + writer.finish(false); + } - try - { - SSTableReader.open(desc, metadata); - fail("SSTR validation should have caught the out-of-order rows"); + try + { + SSTableReader.open(desc, cfs.metadata); + fail("SSTR validation should have caught the out-of-order rows"); + } + catch (IllegalStateException ise) + { /* this is expected */ } + + // open without validation for scrubbing + Set components = new HashSet<>(); + if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists()) + components.add(Component.COMPRESSION_INFO); + components.add(Component.DATA); + components.add(Component.PRIMARY_INDEX); + components.add(Component.FILTER); + components.add(Component.STATS); + components.add(Component.SUMMARY); + components.add(Component.TOC); + + SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs); + if (sstable.last.compareTo(sstable.first) < 0) + sstable.last = sstable.first; + + try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); + Scrubber scrubber = new Scrubber(cfs, txn, false, true, true)) + { + scrubber.scrub(); + } + cfs.loadNewSSTables(); + assertOrderedAll(cfs, 7); + sstable.selfRef().release(); } - catch (IllegalStateException ise) { /* this is expected */ } - - // open without validation for scrubbing - Set components = new HashSet<>(); - //components.add(Component.COMPRESSION_INFO); - components.add(Component.DATA); - components.add(Component.PRIMARY_INDEX); - components.add(Component.FILTER); - components.add(Component.STATS); - components.add(Component.SUMMARY); - components.add(Component.TOC); - - SSTableReader sstable = SSTableReader.openNoValidation(desc, components, cfs); - if (sstable.last.compareTo(sstable.first) < 0) - sstable.last = sstable.first; - - try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, sstable); - Scrubber scrubber = new Scrubber(cfs, txn, false, new OutputHandler.LogOutput(), true, true, true)) + finally { - scrubber.scrub(); + FileUtils.deleteRecursive(tempDataDir); + // reset partitioner + DatabaseDescriptor.setPartitioner(oldPartitioner); } - cfs.loadNewSSTables(); - assertOrderedAll(cfs, 7); - sstable.selfRef().release(); } private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException