cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject svn commit: r1094780 - in /cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra: db/Column.java db/ColumnFamily.java db/DeletedColumn.java db/ExpiringColumn.java db/IColumn.java db/SuperColumn.java io/sstable/SSTableIdentityIterator.java
Date Mon, 18 Apr 2011 21:44:01 GMT
Author: slebresne
Date: Mon Apr 18 21:44:00 2011
New Revision: 1094780

URL: http://svn.apache.org/viewvc?rev=1094780&view=rev
Log:
Make scrub validate column fields
patch by slebresne; reviewed by jbellis for CASSANDRA-2460

Modified:
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
    cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java Mon Apr
18 21:44:00 2011
@@ -26,7 +26,9 @@ import java.util.Collection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
@@ -232,5 +234,19 @@ public class Column implements IColumn
     {
         return !isMarkedForDelete();
     }
+
+    protected void validateName(CFMetaData metadata) throws MarshalException
+    {
+        AbstractType nameValidator = metadata.cfType == ColumnFamilyType.Super ? metadata.subcolumnComparator
: metadata.comparator;
+        nameValidator.validate(name());
+    }
+
+    public void validateFields(CFMetaData metadata) throws MarshalException
+    {
+        validateName(metadata);
+        AbstractType valueValidator = metadata.getValueValidator(name());
+        if (valueValidator != null)
+            valueValidator.validate(value());
+    }
 }
 

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java Mon
Apr 18 21:44:00 2011
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.CFMet
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.filter.QueryPath;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.io.IColumnSerializer;
 import org.apache.cassandra.io.util.IIterableColumns;
 import org.apache.cassandra.utils.FBUtilities;
@@ -424,4 +425,18 @@ public class ColumnFamily implements ICo
         remove(column.name());
         addColumn(column.deepCopy());
     }
+
+    /**
+     * Goes over all columns and check the fields are valid (as far as we can
+     * tell).
+     * This is used to detect corruption after deserialization.
+     */
+    public void validateColumnFields() throws MarshalException
+    {
+        CFMetaData metadata = metadata();
+        for (IColumn column : getSortedColumns())
+        {
+            column.validateFields(metadata);
+        }
+    }
 }

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java Mon
Apr 18 21:44:00 2011
@@ -23,6 +23,8 @@ import java.nio.ByteBuffer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
 public class DeletedColumn extends Column
@@ -62,4 +64,14 @@ public class DeletedColumn extends Colum
     {
         return new DeletedColumn(ByteBufferUtil.clone(name), ByteBufferUtil.clone(value),
timestamp);
     }
+
+    @Override
+    public void validateFields(CFMetaData metadata) throws MarshalException
+    {
+        validateName(metadata);
+        if (value().remaining() != 4)
+            throw new MarshalException("A tombstone value should be 4 bytes long");
+        if (getLocalDeletionTime() < 0)
+            throw new MarshalException("The local deletion time should not be negative");
+    }
 }

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
(original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
Mon Apr 18 21:44:00 2011
@@ -24,7 +24,9 @@ import java.security.MessageDigest;
 
 import org.apache.log4j.Logger;
 
+import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.io.util.DataOutputBuffer;
 import org.apache.cassandra.utils.ByteBufferUtil;
 
@@ -135,4 +137,14 @@ public class ExpiringColumn extends Colu
             throw new IllegalStateException("column is not marked for delete");
         }
     }
+
+    @Override
+    public void validateFields(CFMetaData metadata) throws MarshalException
+    {
+        super.validateFields(metadata);
+        if (timeToLive <= 0)
+            throw new MarshalException("A column TTL should be > 0");
+        if (localExpirationTime < 0)
+            throw new MarshalException("The local expiration time should not be negative");
+    }
 }

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java Mon Apr
18 21:44:00 2011
@@ -22,7 +22,9 @@ import java.nio.ByteBuffer;
 import java.security.MessageDigest;
 import java.util.Collection;
 
+import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.utils.FBUtilities;
 
 public interface IColumn
@@ -45,6 +47,7 @@ public interface IColumn
     public void updateDigest(MessageDigest digest);
     public int getLocalDeletionTime(); // for tombstone GC, so int is sufficient granularity
     public String getString(AbstractType comparator);
+    public void validateFields(CFMetaData metadata) throws MarshalException;
 
     /** clones the column, making copies of any underlying byte buffers */
     IColumn deepCopy();

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java (original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java Mon
Apr 18 21:44:00 2011
@@ -32,7 +32,9 @@ import java.util.concurrent.atomic.Atomi
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.io.IColumnSerializer;
 import org.apache.cassandra.io.util.ColumnSortedMap;
 import org.apache.cassandra.io.util.DataOutputBuffer;
@@ -306,6 +308,15 @@ public class SuperColumn implements ICol
     {
         throw new UnsupportedOperationException("This operation is unsupported on super columns.");
     }
+
+    public void validateFields(CFMetaData metadata) throws MarshalException
+    {
+        metadata.comparator.validate(name());
+        for (IColumn column : getSubColumns())
+        {
+            column.validateFields(metadata);
+        }
+    }
 }
 
 class SuperColumnSerializer implements IColumnSerializer

Modified: cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
(original)
+++ cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
Mon Apr 18 21:44:00 2011
@@ -34,6 +34,7 @@ import org.apache.cassandra.db.ColumnFam
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.IColumn;
 import org.apache.cassandra.db.columniterator.IColumnIterator;
+import org.apache.cassandra.db.marshal.MarshalException;
 import org.apache.cassandra.io.util.BufferedRandomAccessFile;
 import org.apache.cassandra.utils.Filter;
 
@@ -55,6 +56,8 @@ public class SSTableIdentityIterator imp
     // Used by lazilyCompactedRow, so that we see the same things when deserializing the
first and second time
     private final int expireBefore;
 
+    private final boolean validateColumns;
+
     /**
      * Used to iterate through the columns of a row.
      * @param sstable SSTable we are reading ffrom.
@@ -70,7 +73,17 @@ public class SSTableIdentityIterator imp
         this(sstable, file, key, dataStart, dataSize, false);
     }
 
-    public SSTableIdentityIterator(SSTableReader sstable, BufferedRandomAccessFile file,
DecoratedKey key, long dataStart, long dataSize, boolean deserializeRowHeader)
+    /**
+     * Used to iterate through the columns of a row.
+     * @param sstable SSTable we are reading ffrom.
+     * @param file Reading using this file.
+     * @param key Key of this row.
+     * @param dataStart Data for this row starts at this pos.
+     * @param dataSize length of row data
+     * @param checkData if true, do its best to deserialize and check the coherence of row
data
+     * @throws IOException
+     */
+    public SSTableIdentityIterator(SSTableReader sstable, BufferedRandomAccessFile file,
DecoratedKey key, long dataStart, long dataSize, boolean checkData)
     throws IOException
     {
         this.sstable = sstable;
@@ -79,12 +92,13 @@ public class SSTableIdentityIterator imp
         this.dataStart = dataStart;
         this.dataSize = dataSize;
         this.expireBefore = (int)(System.currentTimeMillis() / 1000);
+        this.validateColumns = checkData;
         finishedAt = dataStart + dataSize;
 
         try
         {
             file.seek(this.dataStart);
-            if (deserializeRowHeader)
+            if (checkData)
             {
                 try
                 {
@@ -141,12 +155,19 @@ public class SSTableIdentityIterator imp
     {
         try
         {
-            return sstable.getColumnSerializer().deserialize(file, expireBefore);
+            IColumn column = sstable.getColumnSerializer().deserialize(file, expireBefore);
+            if (validateColumns)
+                column.validateFields(sstable.metadata);
+            return column;
         }
         catch (IOException e)
         {
             throw new IOError(e);
         }
+        catch (MarshalException e)
+        {
+            throw new IOError(new IOException("Error validating row " + key, e));
+        }
     }
 
     public void remove()
@@ -178,6 +199,17 @@ public class SSTableIdentityIterator imp
         file.seek(columnPosition - 4); // seek to before column count int
         ColumnFamily cf = columnFamily.cloneMeShallow();
         ColumnFamily.serializer().deserializeColumns(file, cf);
+        if (validateColumns)
+        {
+            try
+            {
+                cf.validateColumnFields();
+            }
+            catch (MarshalException e)
+            {
+                throw new IOException("Error validating row " + key, e);
+            }
+        }
         return cf;
     }
 



Mime
View raw message