cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [2/6] cassandra git commit: Deserialize sstable metadata in nodetool verify
Date Wed, 04 Oct 2017 06:23:05 GMT
Deserialize sstable metadata in nodetool verify

Patch by marcuse; reviewed by Jason Brown for CASSANDRA-13922


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e400b976
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e400b976
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e400b976

Branch: refs/heads/cassandra-3.11
Commit: e400b976751110d41405bac614189152bf88f7ef
Parents: b32a9e6
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Mon Oct 2 10:11:17 2017 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Wed Oct 4 08:15:23 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  4 +++
 .../cassandra/db/compaction/Verifier.java       | 32 ++++++++++++++++----
 .../org/apache/cassandra/db/VerifyTest.java     | 25 +++++++++++++++
 3 files changed, 55 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/e400b976/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d6423b4..df05f7f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,7 @@
+3.0.16
+  * Deserialise sstable metadata in nodetool verify (CASSANDRA-13922)
+
+
 3.0.15
  * Improve TRUNCATE performance (CASSANDRA-13909)
  * Implement short read protection on partition boundaries (CASSANDRA-13595)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e400b976/src/java/org/apache/cassandra/db/compaction/Verifier.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java
index 88bc3a7..68088b3 100644
--- a/src/java/org/apache/cassandra/db/compaction/Verifier.java
+++ b/src/java/org/apache/cassandra/db/compaction/Verifier.java
@@ -26,13 +26,15 @@ import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.ValidationMetadata;
 import org.apache.cassandra.io.util.DataIntegrityMetadata;
 import org.apache.cassandra.io.util.DataIntegrityMetadata.FileDigestValidator;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.io.util.RandomAccessReader;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.UUIDGen;
 
@@ -58,7 +60,6 @@ public class Verifier implements Closeable
     private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
 
     private int goodRows;
-    private int badRows;
 
     private final OutputHandler outputHandler;
     private FileDigestValidator validator;
@@ -89,6 +90,20 @@ public class Verifier implements Closeable
         long rowStart = 0;
 
         outputHandler.output(String.format("Verifying %s (%s bytes)", sstable, dataFile.length()));
+        outputHandler.output(String.format("Deserializing sstable metadata for %s ", sstable));
+        try
+        {
+            EnumSet<MetadataType> types = EnumSet.of(MetadataType.VALIDATION, MetadataType.STATS,
MetadataType.HEADER);
+            Map<MetadataType, MetadataComponent> sstableMetadata = sstable.descriptor.getMetadataSerializer().deserialize(sstable.descriptor,
types);
+            if (sstableMetadata.containsKey(MetadataType.VALIDATION) &&
+                !((ValidationMetadata)sstableMetadata.get(MetadataType.VALIDATION)).partitioner.equals(sstable.getPartitioner().getClass().getCanonicalName()))
+                throw new IOException("Partitioner does not match validation metadata");
+        }
+        catch (Throwable t)
+        {
+            outputHandler.debug(t.getMessage());
+            markAndThrow(false);
+        }
         outputHandler.output(String.format("Checking computed hash of %s ", sstable));
 
 
@@ -187,7 +202,7 @@ public class Verifier implements Closeable
                     if (key == null || dataSize > dataFile.length())
                         markAndThrow();
 
-                    //mimic the scrub read path
+                    //mimic the scrub read path, intentionally unused
                     try (UnfilteredRowIterator iterator = new SSTableIdentityIterator(sstable,
dataFile, key))
                     {
                     }
@@ -204,7 +219,6 @@ public class Verifier implements Closeable
                 }
                 catch (Throwable th)
                 {
-                    badRows++;
                     markAndThrow();
                 }
             }
@@ -235,8 +249,14 @@ public class Verifier implements Closeable
 
     private void markAndThrow() throws IOException
     {
-        sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor, ActiveRepairService.UNREPAIRED_SSTABLE);
-        throw new CorruptSSTableException(new Exception(String.format("Invalid SSTable %s,
please force repair", sstable.getFilename())), sstable.getFilename());
+        markAndThrow(true);
+    }
+
+    private void markAndThrow(boolean mutateRepaired) throws IOException
+    {
+        if (mutateRepaired) // if we are able to mutate repaired flag, an incremental repair
should be enough
+            sstable.descriptor.getMetadataSerializer().mutateRepairedAt(sstable.descriptor,
ActiveRepairService.UNREPAIRED_SSTABLE);
+        throw new CorruptSSTableException(new Exception(String.format("Invalid SSTable %s,
please force %srepair", sstable.getFilename(), mutateRepaired ? "" : "a full ")), sstable.getFilename());
     }
 
     public CompactionInfo.Holder getVerifyInfo()

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e400b976/test/unit/org/apache/cassandra/db/VerifyTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/VerifyTest.java b/test/unit/org/apache/cassandra/db/VerifyTest.java
index 9de01c1..fc87520 100644
--- a/test/unit/org/apache/cassandra/db/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/db/VerifyTest.java
@@ -341,9 +341,34 @@ public class VerifyTest
             }
             fail("Expected a CorruptSSTableException to be thrown");
         }
+    }
+
+    @Test(expected = CorruptSSTableException.class)
+    public void testVerifyBrokenSSTableMetadata() throws IOException, WriteTimeoutException
+    {
+        CompactionManager.instance.disableAutoCompaction();
+        Keyspace keyspace = Keyspace.open(KEYSPACE);
+        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CORRUPT_CF2);
+
+        fillCF(cfs, 2);
+
+        Util.getAll(Util.cmd(cfs).build());
 
+        SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+
+        String filenameToCorrupt = sstable.descriptor.filenameFor(Component.STATS);
+        RandomAccessFile file = new RandomAccessFile(filenameToCorrupt, "rw");
+        file.seek(0);
+        file.writeBytes(StringUtils.repeat('z', 2));
+        file.close();
+
+        try (Verifier verifier = new Verifier(cfs, sstable, false))
+        {
+            verifier.verify(false);
+        }
     }
 
+
     protected void fillCF(ColumnFamilyStore cfs, int partitionsPerSSTable)
     {
         for (int i = 0; i < partitionsPerSSTable; i++)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message