jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1368520 - in /jackrabbit/oak/trunk/oak-mk/src: main/java/org/apache/jackrabbit/mk/blobs/ test/java/org/apache/jackrabbit/mk/blobs/
Date Thu, 02 Aug 2012 15:17:26 GMT
Author: thomasm
Date: Thu Aug  2 15:17:26 2012
New Revision: 1368520

URL: http://svn.apache.org/viewvc?rev=1368520&view=rev
Log:
OAK-209 BlobStore: use SHA-256 instead of SHA-1, and use two directory levels for FileBlobStore

Modified:
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
    jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
    jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
    jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
    jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStore.java
Thu Aug  2 15:17:26 2012
@@ -65,11 +65,13 @@ import java.util.WeakHashMap;
  */
 public abstract class AbstractBlobStore implements Closeable, BlobStore, Cache.Backend<AbstractBlobStore.BlockId,
AbstractBlobStore.Data> {
 
-    protected static final String HASH_ALGORITHM = "SHA-1";
+    protected static final String HASH_ALGORITHM = "SHA-256";
 
     protected static final int TYPE_DATA = 0;
     protected static final int TYPE_HASH = 1;
     protected static final int TYPE_HASH_COMPRESSED = 2;
+    
+    protected static final int BLOCK_SIZE_LIMIT = 48;
 
     protected Map<String, WeakReference<String>> inUse =
         Collections.synchronizedMap(new WeakHashMap<String, WeakReference<String>>());
@@ -78,7 +80,7 @@ public abstract class AbstractBlobStore 
      * The minimum size of a block. Smaller blocks are inlined (the data store id
      * is the data itself).
      */
-    private int blockSizeMin = 256;
+    private int blockSizeMin = 4096;
 
     /**
      * The size of a block. 128 KB has been found to be as fast as larger
@@ -89,6 +91,7 @@ public abstract class AbstractBlobStore 
     private Cache<AbstractBlobStore.BlockId, Data> cache = Cache.newInstance(this,
8 * 1024 * 1024);
 
     public void setBlockSizeMin(int x) {
+        validateBlockSize(x);
         this.blockSizeMin = x;
     }
 
@@ -97,14 +100,23 @@ public abstract class AbstractBlobStore 
     }
 
     public void setBlockSize(int x) {
+        validateBlockSize(x);
         this.blockSize = x;
     }
+    
+    private static void validateBlockSize(int x) {
+        if (x < BLOCK_SIZE_LIMIT) {
+            throw new IllegalArgumentException(
+                    "The minimum size must be bigger " + 
+                    "than a content hash itself; limit = " + BLOCK_SIZE_LIMIT);
+        }
+    }
 
     public int getBlockSize() {
         return blockSize;
     }
 
-    public String addBlob(String tempFilePath) throws Exception {
+    public String writeBlob(String tempFilePath) throws Exception {
         File file = new File(tempFilePath);
         InputStream in = null;
         try {
@@ -273,13 +285,25 @@ public abstract class AbstractBlobStore 
     }
 
     public Data load(BlockId id) {
+        byte[] data;
         try {
-            return new Data(readBlockFromBackend(id));
+            data = readBlockFromBackend(id);
         } catch (Exception e) {
-            throw new RuntimeException("failed to read block from backend", e);
+            throw new RuntimeException("failed to read block from backend, id " + id, e);
+        }
+        if (data == null) {
+            throw new IllegalArgumentException("The block with id " + id + " was not found");
         }
+        return new Data(data);
     }
 
+    /**
+     * Load the block from the storage backend. Returns null if the block was
+     * not found.
+     * 
+     * @param id the block id
+     * @return the block data, or null
+     */
     protected abstract byte[] readBlockFromBackend(BlockId id) throws Exception;
 
     public long getBlobLength(String blobId) throws IOException {

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/BlobStore.java
Thu Aug  2 15:17:26 2012
@@ -31,7 +31,7 @@ public interface BlobStore {
      * @param tempFilePath the temporary file
      * @return the blob id
      */
-    String addBlob(String tempFilePath) throws Exception;
+    String writeBlob(String tempFilePath) throws Exception;
 
     /**
      * Write a blob from an input stream.
@@ -42,10 +42,31 @@ public interface BlobStore {
      */
     String writeBlob(InputStream in) throws Exception;
 
+    /**
+     * Read a number of bytes from a blob.
+     * 
+     * @param blobId the blob id
+     * @param pos the position within the blob
+     * @param buff the target byte array
+     * @param off the offset within the target array
+     * @param length the number of bytes to read
+     * @return the number of bytes read
+     */
     int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws Exception;
 
+    /**
+     * Get the length of the blob.
+     * 
+     * @param blobId the blob id
+     * @return the length
+     */
     long getBlobLength(String blobId) throws Exception;
 
+    /**
+     * Close all internally used resources, such as file handles. This method
+     * should be called at the end of the components lifecycle. After calling
+     * this method, the blob store should no longer be used.
+     */
     void close();
 
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/FileBlobStore.java
Thu Aug  2 15:17:26 2012
@@ -48,7 +48,7 @@ public class FileBlobStore extends Abstr
     }
 
     @Override
-    public String addBlob(String tempFilePath) throws Exception {
+    public String writeBlob(String tempFilePath) throws Exception {
         File file = new File(tempFilePath);
         InputStream in = new FileInputStream(file);
         MessageDigest messageDigest = MessageDigest.getInstance(HASH_ALGORITHM);
@@ -106,11 +106,12 @@ public class FileBlobStore extends Abstr
 
     private File getFile(byte[] digest, boolean old) {
         String id = StringUtils.convertBytesToHex(digest);
-        String sub = id.substring(id.length() - 2);
+        String sub1 = id.substring(id.length() - 2);
+        String sub2 = id.substring(id.length() - 4, id.length() - 2);
         if (old) {
-            sub += OLD_SUFFIX;
+            sub2 += OLD_SUFFIX;
         }
-        return new File(new File(baseDir, sub), id + ".dat");
+        return new File(new File(new File(baseDir, sub1), sub2), id + ".dat");
     }
 
     @Override
@@ -137,19 +138,23 @@ public class FileBlobStore extends Abstr
     @Override
     public void startMark() throws Exception {
         mark = true;
-        for (int i = 0; i < 256; i++) {
-            String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
-            File d = new File(baseDir, sub);
-            File old = new File(baseDir, sub + OLD_SUFFIX);
-            if (d.exists()) {
-                if (old.exists()) {
-                    for (File p : d.listFiles()) {
-                        String name = p.getName();
-                        File newName = new File(old, name);
-                        p.renameTo(newName);
+        for (int j = 0; j < 256; j++) {
+            String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j });
+            File x = new File(baseDir, sub1);
+            for (int i = 0; i < 256; i++) {
+                String sub2 = StringUtils.convertBytesToHex(new byte[] { (byte) i });
+                File d = new File(x, sub2);
+                File old = new File(x, sub2 + OLD_SUFFIX);
+                if (d.exists()) {
+                    if (old.exists()) {
+                        for (File p : d.listFiles()) {
+                            String name = p.getName();
+                            File newName = new File(old, name);
+                            p.renameTo(newName);
+                        }
+                    } else {
+                        d.renameTo(old);
                     }
-                } else {
-                    d.renameTo(old);
                 }
             }
         }
@@ -175,17 +180,21 @@ public class FileBlobStore extends Abstr
     @Override
     public int sweep() throws IOException {
         int count = 0;
-        for (int i = 0; i < 256; i++) {
-            String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
-            File old = new File(baseDir, sub + OLD_SUFFIX);
-            if (old.exists()) {
-                for (File p : old.listFiles()) {
-                    String name = p.getName();
-                    File file = new File(old, name);
-                    file.delete();
-                    count++;
+        for (int j = 0; j < 256; j++) {
+            String sub1 = StringUtils.convertBytesToHex(new byte[] { (byte) j });
+            File x = new File(baseDir, sub1);
+            for (int i = 0; i < 256; i++) {
+                String sub = StringUtils.convertBytesToHex(new byte[] { (byte) i });
+                File old = new File(x, sub + OLD_SUFFIX);
+                if (old.exists()) {
+                    for (File p : old.listFiles()) {
+                        String name = p.getName();
+                        File file = new File(old, name);
+                        file.delete();
+                        count++;
+                    }
+                    old.delete();
                 }
-                old.delete();
             }
         }
         mark = false;

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStore.java
Thu Aug  2 15:17:26 2012
@@ -29,7 +29,11 @@ public class MemoryBlobStore extends Abs
 
     @Override
     protected byte[] readBlockFromBackend(BlockId id) {
-        return map.get(id);
+        byte[] result = map.get(id);
+        if (result == null) {
+            result = old.get(id);
+        }
+        return result;
     }
 
     @Override

Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/AbstractBlobStoreTest.java
Thu Aug  2 15:17:26 2012
@@ -61,7 +61,7 @@ public abstract class AbstractBlobStoreT
         OutputStream out = new FileOutputStream(tempFile, false);
         out.write(data);
         out.close();
-        String s = store.addBlob(tempFileName);
+        String s = store.writeBlob(tempFileName);
         assertEquals(data.length, store.getBlobLength(s));
         byte[] buff = new byte[1];
         for (int i = 0; i < data.length; i += 1024) {
@@ -69,7 +69,7 @@ public abstract class AbstractBlobStoreT
             assertEquals(data[i], buff[0]);
         }
         try {
-            store.addBlob(tempFileName + "_wrong");
+            store.writeBlob(tempFileName + "_wrong");
             fail();
         } catch (Exception e) {
             // expected
@@ -162,7 +162,7 @@ public abstract class AbstractBlobStoreT
         HashMap<String, byte[]> map = new HashMap<String, byte[]>();
         ArrayList<String> mem = new ArrayList<String>();
         int count;
-        for (int i = 1; i < 10000; i += (i + 1) * 10) {
+        for (int i = 1; i <= 1000; i *= 10) {
             byte[] data = new byte[i];
             String id;
             id = store.writeBlob(new ByteArrayInputStream(data));
@@ -194,7 +194,7 @@ public abstract class AbstractBlobStoreT
             }
             store.mark(id);
         }
-        store.sweep();
+        count = store.sweep();
 
         store.clearInUse();
         store.clearCache();

Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/DbBlobStoreTest.java
Thu Aug  2 15:17:26 2012
@@ -35,7 +35,7 @@ public class DbBlobStoreTest extends Abs
         DbBlobStore blobStore = new DbBlobStore();
         blobStore.setConnectionPool(cp);
         blobStore.setBlockSize(128);
-        blobStore.setBlockSizeMin(32);
+        blobStore.setBlockSizeMin(48);
         this.store = blobStore;
     }
 

Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/FileBlobStoreTest.java
Thu Aug  2 15:17:26 2012
@@ -24,7 +24,7 @@ public class FileBlobStoreTest extends A
     public void setUp() throws Exception {
         FileBlobStore store = new FileBlobStore("target/temp");
         store.setBlockSize(128);
-        store.setBlockSizeMin(32);
+        store.setBlockSizeMin(48);
         this.store = store;
     }
 

Modified: jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java?rev=1368520&r1=1368519&r2=1368520&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/test/java/org/apache/jackrabbit/mk/blobs/MemoryBlobStoreTest.java
Thu Aug  2 15:17:26 2012
@@ -23,6 +23,8 @@ public class MemoryBlobStoreTest extends
 
     public void setUp() throws Exception {
         store = new MemoryBlobStore();
-    }
+        store.setBlockSize(128);
+        store.setBlockSizeMin(48);
+    }    
 
 }



Mime
View raw message