cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From slebre...@apache.org
Subject svn commit: r1199763 - in /cassandra/branches/cassandra-1.0: ./ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/io/util/
Date Wed, 09 Nov 2011 14:04:40 GMT
Author: slebresne
Date: Wed Nov  9 14:04:40 2011
New Revision: 1199763

URL: http://svn.apache.org/viewvc?rev=1199763&view=rev
Log:
Compute sha1 sum for uncompressed data file
patch by slebresne; reviewed by jbellis for CASSANDRA-3456

Modified:
    cassandra/branches/cassandra-1.0/CHANGES.txt
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/util/SequentialWriter.java

Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1199763&r1=1199762&r2=1199763&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Wed Nov  9 14:04:40 2011
@@ -4,6 +4,7 @@
  * (CQL) fix handling of rows with no columns (CASSANDRA-3424, 3473)
  * fix querying supercolumns by name returning only a subset of
    subcolumns or old subcolumn versions (CASSANDRA-3446)
+ * automatically compute sha1 sum for uncompressed data files (CASSANDRA-3456)
 Merged from 0.8:
  * Make counter shard merging thread safe (CASSANDRA-3178)
  * fix updating CF row_cache_provider (CASSANDRA-3414)

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java?rev=1199763&r1=1199762&r2=1199763&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
Wed Nov  9 14:04:40 2011
@@ -54,7 +54,9 @@ public class Component
         // statistical metadata about the content of the sstable
         STATS("Statistics.db"),
         // a bitmap secondary index: many of these may exist per sstable
-        BITMAP_INDEX("Bitidx.db");
+        BITMAP_INDEX("Bitidx.db"),
+        // holds sha1 sum of the data file (to be checked by sha1sum)
+        DIGEST("Digest.sha1");
 
         final String repr;
         Type(String repr)
@@ -78,6 +80,7 @@ public class Component
     public final static Component COMPACTED_MARKER = new Component(Type.COMPACTED_MARKER,
-1);
     public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO,
-1);
     public final static Component STATS = new Component(Type.STATS, -1);
+    public final static Component DIGEST = new Component(Type.DIGEST, -1);
 
     public final Type type;
     public final int id;
@@ -108,6 +111,7 @@ public class Component
             case COMPACTED_MARKER:
             case COMPRESSION_INFO:
             case STATS:
+            case DIGEST:
                 return type.repr;
             case BITMAP_INDEX:
                 return String.format("%d%c%s", id, separator, type.repr);
@@ -145,6 +149,7 @@ public class Component
             case COMPACTED_MARKER:  component = Component.COMPACTED_MARKER; break;
             case COMPRESSION_INFO:  component = Component.COMPRESSION_INFO; break;
             case STATS:             component = Component.STATS;            break;
+            case DIGEST:            component = Component.DIGEST;           break;
             case BITMAP_INDEX:
                  component = new Component(type, id);
                  break;

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1199763&r1=1199762&r2=1199763&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Wed Nov  9 14:04:40 2011
@@ -58,6 +58,7 @@ public abstract class SSTable
     public static final String COMPONENT_INDEX = Component.Type.PRIMARY_INDEX.repr;
     public static final String COMPONENT_FILTER = Component.Type.FILTER.repr;
     public static final String COMPONENT_STATS = Component.Type.STATS.repr;
+    public static final String COMPONENT_DIGEST = Component.Type.DIGEST.repr;
 
     public static final String TEMPFILE_MARKER = "tmp";
 

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199763&r1=1199762&r2=1199763&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Wed Nov  9 14:04:40 2011
@@ -37,9 +37,7 @@ import org.apache.cassandra.dht.IPartiti
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 import org.apache.cassandra.io.util.*;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.BloomFilter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.*;
 
 public class SSTableWriter extends SSTable
 {
@@ -66,6 +64,10 @@ public class SSTableWriter extends SSTab
         Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA,
Component.FILTER, Component.PRIMARY_INDEX, Component.STATS));
         if (metadata.compressionParameters().sstableCompressor != null)
             components.add(Component.COMPRESSION_INFO);
+        else
+            // it would feel safer to actually add this component later in maybeWriteDigest(),
+            // but the components are unmodifiable after construction
+            components.add(Component.DIGEST);
         return components;
     }
 
@@ -94,6 +96,7 @@ public class SSTableWriter extends SSTab
         {
             dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
             dataFile = SequentialWriter.open(new File(getFilename()), true);
+            dataFile.setComputeDigest();
         }
 
         this.sstableMetadataCollector = sstableMetadataCollector;
@@ -300,6 +303,7 @@ public class SSTableWriter extends SSTab
         // write sstable statistics
         SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata();
         writeMetadata(descriptor, sstableMetadata);
+        maybeWriteDigest();
 
         // remove the 'tmp' marker from all components
         final Descriptor newdesc = rename(descriptor, components);
@@ -324,6 +328,21 @@ public class SSTableWriter extends SSTab
         return sstable;
     }
 
+    private void maybeWriteDigest() throws IOException
+    {
+        byte[] digest = dataFile.digest();
+        if (digest == null)
+            return;
+
+        SequentialWriter out = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_DIGEST)),
true);
+        // Writting output compatible with sha1sum
+        Descriptor newdesc = descriptor.asTemporary(false);
+        String[] tmp = newdesc.filenameFor(SSTable.COMPONENT_DATA).split(new Character(File.separatorChar).toString());
+        String dataFileName = tmp[tmp.length - 1];
+        out.write(String.format("%s  %s", Hex.bytesToHex(digest), dataFileName).getBytes());
+        out.close();
+    }
+
     private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata) throws
IOException
     {
         SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)),
true);

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/util/SequentialWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/util/SequentialWriter.java?rev=1199763&r1=1199762&r2=1199763&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/util/SequentialWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/util/SequentialWriter.java
Wed Nov  9 14:04:40 2011
@@ -20,6 +20,8 @@ package org.apache.cassandra.io.util;
 
 import java.io.*;
 import java.nio.channels.ClosedChannelException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
 import org.apache.cassandra.utils.CLibrary;
 
@@ -47,6 +49,7 @@ public class SequentialWriter extends Ou
     private long ioCacheStartOffset = 0, bytesSinceCacheFlush = 0;
 
     public final DataOutputStream stream;
+    private MessageDigest digest;
 
     public SequentialWriter(File file, int bufferSize, boolean skipIOCache) throws IOException
     {
@@ -199,6 +202,8 @@ public class SequentialWriter extends Ou
     protected void flushData() throws IOException
     {
         out.write(buffer, 0, validBufferBytes);
+        if (digest != null)
+            digest.update(buffer, 0, validBufferBytes);
     }
 
     public long getFilePointer()
@@ -286,6 +291,41 @@ public class SequentialWriter extends Ou
     }
 
     /**
+     * Turn on digest computation on this writer.
+     * This can only be called before any data is written to this write,
+     * otherwise an IllegalStateException is thrown.
+     */
+    public void setComputeDigest()
+    {
+        if (current != 0)
+            throw new IllegalStateException();
+
+        try
+        {
+            digest = MessageDigest.getInstance("SHA-1");
+        }
+        catch (NoSuchAlgorithmException e)
+        {
+            // SHA-1 is standard in java 6
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Return the digest associated to this file or null if no digest was
+     * created.
+     * This can only be called once the file is fully created, i.e. after
+     * close() has been called. Otherwise an IllegalStateException is thrown.
+     */
+    public byte[] digest()
+    {
+        if (buffer != null)
+            throw new IllegalStateException();
+
+        return digest == null ? null : digest.digest();
+    }
+
+    /**
      * Class to hold a mark to the position of the file
      */
     protected static class BufferedFileWriterMark implements FileMark



Mime
View raw message