jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1155401 - in /jackrabbit/sandbox/microkernel/src: main/java/org/apache/jackrabbit/mk/datastore/ main/java/org/apache/jackrabbit/mk/mem/ main/java/org/apache/jackrabbit/mk/store/ main/java/org/apache/jackrabbit/mk/util/ test/java/org/apache...
Date Tue, 09 Aug 2011 15:25:56 GMT
Author: thomasm
Date: Tue Aug  9 15:25:56 2011
New Revision: 1155401

URL: http://svn.apache.org/viewvc?rev=1155401&view=rev
Log:
Data store: split entries into smaller blocks (could be backported to Jackrabbit 2 if required)

Added:
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/DbStore.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/IOUtils.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/SmallLRUCache.java
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DbStoreTest.java
Modified:
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/H2RevisionStore.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DataStoreTest.java
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java

Added: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/DbStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/DbStore.java?rev=1155401&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/DbStore.java (added)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/datastore/DbStore.java Tue Aug  9 15:25:56 2011
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.datastore;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.jackrabbit.mk.util.IOUtils;
+import org.apache.jackrabbit.mk.util.StringUtils;
+import org.h2.jdbcx.JdbcConnectionPool;
+import org.h2.util.SmallLRUCache;
+
+/**
+ * A database data store that splits the binaries in relatively small blocks.
+ * Each data store id a list of zero or more entries. Each entry is either
+ * <ul>
+ * <li>data (a number of bytes), or</li>
+ * <li>the hash code of the content of a number of bytes, or</li>
+ * <li>the hash code of the content of a data store id (indirect hash)</li>
+ * </ul>
+ * Thanks to the indirection, blocks can be kept relatively small, so that
+ * caching is simpler, and so that the storage backend doesn't need to support
+ * arbitrary size blobs (some storage backends buffer blobs in memory) and fast
+ * seeks (some storage backends re-read the whole blob when seeking).
+ * <p>
+ * The the format of a 'data' entry is: type (one byte; 0 for data), length
+ * (variable size int), data (bytes).
+ * <p>
+ * The format of a 'hash of content' entry is: type (one byte; 1 for hash),
+ * level (variable size int, 0 meaning not nested), size (variable size long),
+ * hash code length (variable size int), hash code.
+ * <p>
+ * The format of a 'hash of data store id' entry is: type (one byte; 1 for
+ * hash), level (variable size int, nesting level), total size (variable size
+ * long), size of data store id (variable size long), hash code length (variable
+ * size int), hash code.
+ */
+public class DbStore {
+
+    private static final int TYPE_DATA = 0;
+    private static final int TYPE_HASH = 1;
+
+    /**
+     * The minimum size of a block. Smaller blocks are stored (the data store id
+     * is the data itself).
+     */
+    private int blockSizeMin = 256;
+
+    /**
+     * The size of a block. This number has been found to be as fast as larger
+     * values, and faster than smaller values.
+     */
+    private int blockSize = 128 * 1024;
+
+    private static final String HASH_ALGORITHM = "SHA-1";
+
+    public void setConnectionPool(JdbcConnectionPool cp) throws SQLException {
+        this.cp = cp;
+        Connection conn = cp.getConnection();
+        Statement stat = conn.createStatement();
+        stat.execute("create table if not exists datastore_meta" +
+                "(id varchar primary key, level int, lastMod bigint)");
+        stat.execute("create table if not exists datastore_data" +
+                "(id varchar primary key, data binary)");
+        conn.close();
+    }
+
+    public void setBlockSizeMin(int x) {
+        this.blockSizeMin = x;
+    }
+
+    public void setBlockSize(int x) {
+        this.blockSize = x;
+    }
+
+    private JdbcConnectionPool cp;
+    private SmallLRUCache<String, byte[]> datastoreCache = SmallLRUCache.newInstance(5);
+
+    public String writeBlob(InputStream in) throws Exception {
+        Connection conn = cp.getConnection();
+        try {
+            ByteArrayOutputStream idStream = new ByteArrayOutputStream();
+            convertBlobToId(conn, in, idStream, 0, 0);
+            byte[] id = idStream.toByteArray();
+            // System.out.println("    write blob " +  StringUtils.convertBytesToHex(id));
+            return StringUtils.convertBytesToHex(id);
+        } finally {
+            conn.close();
+        }
+    }
+
+    private void convertBlobToId(Connection conn, InputStream in, ByteArrayOutputStream idStream, int level, long totalLength) throws Exception {
+        byte[] block = new byte[blockSize];
+        int count = 0;
+        while (true) {
+            MessageDigest digest = MessageDigest.getInstance(HASH_ALGORITHM);
+            ByteArrayOutputStream buff = new ByteArrayOutputStream();
+            DigestOutputStream dout = new DigestOutputStream(buff, digest);
+            int blockLen = IOUtils.readFully(in, block, 0, block.length);
+            count++;
+            if (blockLen == 0) {
+                break;
+            } else if (blockLen < blockSizeMin) {
+                idStream.write(TYPE_DATA);
+                IOUtils.writeVarInt(idStream, blockLen);
+                idStream.write(block, 0, blockLen);
+                totalLength += blockLen;
+            } else {
+                dout.write(block, 0, blockLen);
+                byte[] blockId = digest.digest();
+                idStream.write(TYPE_HASH);
+                IOUtils.writeVarInt(idStream, level);
+                if (level > 0) {
+                    IOUtils.writeVarLong(idStream, totalLength);
+                }
+                IOUtils.writeVarLong(idStream, blockLen);
+                totalLength += blockLen;
+                IOUtils.writeVarInt(idStream, blockId.length);
+                idStream.write(blockId);
+                byte[] data = buff.toByteArray();
+                storeBlock(conn, blockId, level, data);
+            }
+            if (idStream.size() > blockSize / 2) {
+                // convert large ids to a block, but ensure it can be stored as
+                // one block (otherwise the indirection no longer works)
+                byte[] idBlock = idStream.toByteArray();
+                idStream.reset();
+                convertBlobToId(conn, new ByteArrayInputStream(idBlock), idStream, level + 1, totalLength);
+                count = 1;
+            }
+        }
+        if (count > 0 && idStream.size() > blockSizeMin) {
+            // at the very end, convert large ids to a block,
+            // because large block ids are not handy
+            // (specially if they are used to read data in small chunks)
+            byte[] idBlock = idStream.toByteArray();
+            idStream.reset();
+            convertBlobToId(conn, new ByteArrayInputStream(idBlock), idStream, level + 1, totalLength);
+        }
+    }
+
+    private void storeBlock(Connection conn, byte[] blockId, int level, byte[] data) throws SQLException {
+        String id = StringUtils.convertBytesToHex(blockId);
+        long now = System.currentTimeMillis();
+        PreparedStatement prep = conn.prepareStatement(
+            "update datastore_meta set lastMod = ? where id = ?");
+        int count;
+        try {
+            prep.setLong(1, now);
+            prep.setString(2, id);
+            count = prep.executeUpdate();
+        } finally {
+            prep.close();
+        }
+        if (count == 0) {
+            try {
+                prep = conn.prepareStatement(
+                    "insert into datastore_meta(id, level, lastMod) values(?, ?, ?)");
+                try {
+                    prep.setString(1, id);
+                    prep.setInt(2, level);
+                    prep.setLong(3, now);
+                    prep.execute();
+                } finally {
+                    prep.close();
+                }
+            } catch (SQLException e) {
+                // already exists - ok
+            }
+            try {
+                prep = conn.prepareStatement(
+                    "insert into datastore_data(id, data) values(?, ?)");
+                try {
+                    prep.setString(1, id);
+                    prep.setBytes(2, data);
+                    prep.execute();
+                } finally {
+                    prep.close();
+                }
+            } catch (SQLException e) {
+                // already exists - ok
+            }
+        }
+    }
+
+    public int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws Exception {
+        byte[] id = StringUtils.convertHexToBytes(blobId);
+        ByteArrayInputStream idStream = new ByteArrayInputStream(id);
+        while (true) {
+            int type = idStream.read();
+            if (type == -1) {
+                return -1;
+            } else if (type == TYPE_DATA) {
+                int len = IOUtils.readVarInt(idStream);
+                if (pos < len) {
+                    IOUtils.skipFully(idStream, (int) pos);
+                    len -= pos;
+                    if (length < len) {
+                        len = length;
+                    }
+                    IOUtils.readFully(idStream, buff, off, len);
+                    return len;
+                }
+                IOUtils.skipFully(idStream, len);
+                pos -= len;
+            } else if (type == TYPE_HASH) {
+                int level = IOUtils.readVarInt(idStream);
+                long totalLength = IOUtils.readVarLong(idStream);
+                if (level > 0) {
+                    // block length (ignored)
+                    IOUtils.readVarLong(idStream);
+                }
+                byte[] digest = new byte[IOUtils.readVarInt(idStream)];
+                IOUtils.readFully(idStream, digest, 0, digest.length);
+                if (pos >= totalLength) {
+                    pos -= totalLength;
+                } else {
+                    byte[] block = readBlock(digest);
+                    if (level > 0) {
+                        idStream = new ByteArrayInputStream(block);
+                    } else {
+                        ByteArrayInputStream in = new ByteArrayInputStream(block);
+                        IOUtils.skipFully(in, (int) pos);
+                        return IOUtils.readFully(in, buff, off, length);
+                    }
+                }
+            } else {
+                throw new IOException("Datastore id type " + type + " for blob " + blobId);
+            }
+        }
+    }
+
+    private byte[] readBlock(byte[] digest) throws Exception {
+        String id = StringUtils.convertBytesToHex(digest);
+        byte[] block = datastoreCache.get(id);
+        if (block == null) {
+            block = readBlockFromDb(digest);
+            datastoreCache.put(id, block);
+        }
+        return block;
+    }
+
+    private byte[] readBlockFromDb(byte[] digest) throws Exception {
+        Connection conn = cp.getConnection();
+        try {
+            PreparedStatement prep = conn.prepareStatement(
+                "select data from datastore_data where id = ?");
+            try {
+                String id = StringUtils.convertBytesToHex(digest);
+                prep.setString(1, id);
+                ResultSet rs = prep.executeQuery();
+                if (!rs.next()) {
+                    throw new IOException("Datastore block " + id + " not found");
+                }
+                byte[] data = rs.getBytes(1);
+                // System.out.println("    read block " + id + " blockLen: " + data.length + " [0]: " + data[0]);
+                return data;
+            } finally {
+                prep.close();
+            }
+        } finally {
+            conn.close();
+        }
+    }
+
+    public long getBlobLength(String blobId) throws IOException {
+        byte[] id = StringUtils.convertHexToBytes(blobId);
+        ByteArrayInputStream idStream = new ByteArrayInputStream(id);
+        long totalLength = 0;
+        while (true) {
+            int type = idStream.read();
+            if (type == -1) {
+                break;
+            }
+            if (type == TYPE_DATA) {
+                int len = IOUtils.readVarInt(idStream);
+                IOUtils.skipFully(idStream, len);
+                totalLength += len;
+            } else if (type == TYPE_HASH) {
+                int level = IOUtils.readVarInt(idStream);
+                totalLength += IOUtils.readVarLong(idStream);
+                if (level > 0) {
+                    // block length (ignored)
+                    IOUtils.readVarLong(idStream);
+                }
+                int digestLength = IOUtils.readVarInt(idStream);
+                IOUtils.skipFully(idStream, digestLength);
+            } else {
+                throw new IOException("Datastore id type " + type + " for blob " + blobId);
+            }
+        }
+        return totalLength;
+    }
+
+}

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java Tue Aug  9 15:25:56 2011
@@ -31,6 +31,7 @@ public class NodeImpl {
     private final long revId;
     private HashMap<String, String> properties;
     private LinkedHashMap<String, NodeImpl> childNodes;
+    private String path;
 
     public NodeImpl(long revId) {
         this.revId = revId;
@@ -274,4 +275,12 @@ public class NodeImpl {
         return node;
     }
 
+    public void setPath(String path) {
+        this.path = path;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
 }

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java Tue Aug  9 15:25:56 2011
@@ -18,7 +18,7 @@ package org.apache.jackrabbit.mk.store;
 
 import java.io.DataInputStream;
 import java.io.InputStream;
-import org.apache.jackrabbit.mk.util.StringUtils;
+import org.apache.jackrabbit.mk.util.IOUtils;
 
 /**
  * Immutable Commit object...
@@ -70,10 +70,10 @@ public class Commit extends StoredObject
             if (in.read() != Constants.COMMIT) {
                 throw new Exception("unknown data format");
             }
-            String rootNodeId = StringUtils.readString(in);
+            String rootNodeId = IOUtils.readString(in);
             long commitTS = in.readLong();
-            String msg = StringUtils.readString(in);
-            String parentId = StringUtils.readString(in);
+            String msg = IOUtils.readString(in);
+            String parentId = IOUtils.readString(in);
             return new Commit(id, rootNodeId, commitTS, "".equals(msg) ? null : msg, "".equals(parentId) ? null : parentId);
         } finally {
             in.close();

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/H2RevisionStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/H2RevisionStore.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/H2RevisionStore.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/H2RevisionStore.java Tue Aug  9 15:25:56 2011
@@ -16,27 +16,28 @@
  */
 package org.apache.jackrabbit.mk.store;
 
-import org.apache.jackrabbit.mk.api.MicroKernelException;
-import org.apache.jackrabbit.mk.util.StringUtils;
-import org.h2.jdbcx.JdbcConnectionPool;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
-import java.security.DigestInputStream;
 import java.security.DigestOutputStream;
 import java.security.MessageDigest;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;
+import org.apache.jackrabbit.mk.datastore.DbStore;
+import org.apache.jackrabbit.mk.util.StringUtils;
+import org.h2.jdbcx.JdbcConnectionPool;
 
 /**
  *
  */
 public class H2RevisionStore extends SimpleRevisionStore {
 
-    JdbcConnectionPool cp;
+    private JdbcConnectionPool cp;
+    private DbStore blobStore;
 
     public H2RevisionStore() throws Exception {
         initialized = false;
@@ -65,11 +66,11 @@ public class H2RevisionStore extends Sim
             stmt.execute("create table if not exists REVS (ID binary primary key, DATA binary)");
             stmt.execute("create table if not exists head(id varchar) as select ''");
             stmt.execute("create sequence if not exists datastore_id");
-            stmt.execute("create table if not exists datastore(id varchar primary key, data blob)");
+            blobStore = new DbStore();
+            blobStore.setConnectionPool(cp);
         } finally {
             con.close();
         }
-
         initialized = true;
     }
 
@@ -158,7 +159,7 @@ public class H2RevisionStore extends Sim
             }
         } finally {
             dos.flush();
-            //baos.close();
+            // baos.close();
             in.close();
         }
 
@@ -166,7 +167,8 @@ public class H2RevisionStore extends Sim
         String id = StringUtils.convertBytesToHex(rawId);
         Connection con = cp.getConnection();
         try {
-            PreparedStatement stmt = con.prepareStatement(
+            PreparedStatement stmt = con
+                    .prepareStatement(
                     "insert into REVS (ID, DATA) select ?, ? where not exists (select 1 from revs where ID = ?)");
             try {
                 stmt.setBytes(1, rawId);
@@ -184,114 +186,17 @@ public class H2RevisionStore extends Sim
 
     @Override
     public String writeBlob(InputStream in) throws Exception {
-        if (!initialized) {
-            throw new IllegalStateException("not initialized");
-        }
-        Connection con = cp.getConnection();
-        String id;
-        try {
-            String tempId;
-            PreparedStatement stmt = con.prepareStatement(
-                    "insert into datastore(id, data) values(datastore_id.nextval, ?)");
-            try {
-                MessageDigest digest = MessageDigest.getInstance("SHA-1");
-                DigestInputStream din = new DigestInputStream(in, digest);
-                stmt.setBinaryStream(1, din);
-                stmt.executeUpdate();
-                ResultSet rs = stmt.getGeneratedKeys();
-                rs.next();
-                tempId = rs.getString(1);
-                id = "ds:" + StringUtils.convertBytesToHex(digest.digest());
-            } finally {
-                stmt.close();
-            }
-            stmt = con.prepareStatement(
-                    "update datastore set id = ? where id = ? and not exists(select * from datastore where id = ?)");
-            try {
-                stmt.setString(1, id);
-                stmt.setString(2, tempId);
-                stmt.setString(3, id);
-                int count = stmt.executeUpdate();
-                if (count == 0) {
-                    stmt.close();
-                    stmt = con.prepareStatement(
-                            "delete from datastore where id = ?");
-                    stmt.setString(1, tempId);
-                    stmt.executeUpdate();
-                }
-            } finally {
-                stmt.close();
-            }
-        } finally {
-            con.close();
-        }
-        return id;
+        return blobStore.writeBlob(in);
     }
 
     @Override
     public int readBlob(String blobId, long pos, byte[] buff, int off, int length) throws Exception {
-        if (!initialized) {
-            throw new IllegalStateException("not initialized");
-        }
-        Connection con = cp.getConnection();
-        try {
-            final PreparedStatement stmt = con.prepareStatement(
-                    "select data from datastore where id = ?");
-            try {
-                stmt.setString(1, blobId);
-                ResultSet rs = stmt.executeQuery();
-                if (!rs.next()) {
-                    throw new MicroKernelException("Blob not found: " + blobId);
-                }
-
-                int TODO_WORKAROUND_FOR_H2_BUG;
-                int max = (int) Math.min(length, getBlobLength(blobId) - pos);
-                if (max != length) {
-                    if (max <= 0) {
-                        rs.close();
-                        return -1;
-                    }
-                    length = max;
-                }
-
-                InputStream in = rs.getBinaryStream(1);
-                in.skip(pos);
-                int len = in.read(buff, off, length);
-                rs.close();
-                return len;
-            } finally {
-                stmt.close();
-            }
-        } finally {
-            con.close();
-        }
+        return blobStore.readBlob(blobId, pos, buff, off, length);
     }
 
     @Override
-    public long getBlobLength(String blobId) throws Exception {
-        if (!initialized) {
-            throw new IllegalStateException("not initialized");
-        }
-        // TODO encode length in the blob id
-        Connection con = cp.getConnection();
-        try {
-            final PreparedStatement stmt = con.prepareStatement(
-                    "select length(data) from datastore where id = ?");
-            try {
-                stmt.setString(1, blobId);
-                ResultSet rs = stmt.executeQuery();
-                if (!rs.next()) {
-                    throw new MicroKernelException("Blob not found: " + blobId);
-                }
-                long length = rs.getLong(1);
-                rs.close();
-                return length;
-            } finally {
-                stmt.close();
-            }
-        } finally {
-            con.close();
-        }
+    public long getBlobLength(String blobId) throws IOException {
+        return blobStore.getBlobLength(blobId);
     }
 
 }

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java Tue Aug  9 15:25:56 2011
@@ -21,7 +21,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import org.apache.jackrabbit.mk.util.StringUtils;
+import org.apache.jackrabbit.mk.util.IOUtils;
 
 /**
  *
@@ -82,10 +82,10 @@ public class MutableCommit {
         DataOutputStream out = new DataOutputStream(baos);
         try {
             out.write(Constants.COMMIT);
-            StringUtils.writeString(out, rootNodeId);
+            IOUtils.writeString(out, rootNodeId);
             out.writeLong(commitTS);
-            StringUtils.writeString(out, msg == null ? "" : msg);
-            StringUtils.writeString(out, parentId == null ? "" : parentId);
+            IOUtils.writeString(out, msg == null ? "" : msg);
+            IOUtils.writeString(out, parentId == null ? "" : parentId);
             out.close();
         } catch (IOException e) {
             throw new RuntimeException(e);

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java Tue Aug  9 15:25:56 2011
@@ -18,13 +18,12 @@ package org.apache.jackrabbit.mk.store;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import org.apache.jackrabbit.mk.util.StringUtils;
+import org.apache.jackrabbit.mk.util.IOUtils;
 
 /**
  *
@@ -66,15 +65,15 @@ public class MutableNode {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         try {
             out.write(Constants.NODE);
-            StringUtils.writeVarInt(out, properties.size());
+            IOUtils.writeVarInt(out, properties.size());
             for (Map.Entry<String, String> entry : properties.entrySet()) {
-                StringUtils.writeString(out, entry.getKey());
-                StringUtils.writeString(out, entry.getValue());
+                IOUtils.writeString(out, entry.getKey());
+                IOUtils.writeString(out, entry.getValue());
             }
-            StringUtils.writeVarInt(out, childEntries.size());
+            IOUtils.writeVarInt(out, childEntries.size());
             for (Map.Entry<String, String> entry : childEntries.entrySet()) {
-                StringUtils.writeString(out, entry.getKey());
-                StringUtils.writeString(out, entry.getValue());
+                IOUtils.writeString(out, entry.getKey());
+                IOUtils.writeString(out, entry.getValue());
             }
             out.close();
         } catch (IOException e) {

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java Tue Aug  9 15:25:56 2011
@@ -22,7 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
-import org.apache.jackrabbit.mk.util.StringUtils;
+import org.apache.jackrabbit.mk.util.IOUtils;
 
 /**
  * Immutable Node object...
@@ -71,18 +71,18 @@ public class Node extends StoredObject {
             if (in.read() != Constants.NODE) {
                 throw new Exception("unknown data format");
             }
-            int count = StringUtils.readVarInt(in);
+            int count = IOUtils.readVarInt(in);
             Map<String, String> props = new HashMap<String, String>(count);
             while (count-- > 0) {
-                String name = StringUtils.readString(in);
-                String value = StringUtils.readString(in);
+                String name = IOUtils.readString(in);
+                String value = IOUtils.readString(in);
                 props.put(name, value);
             }
-            count = StringUtils.readVarInt(in);
+            count = IOUtils.readVarInt(in);
             LinkedHashMap<String, String> childEntries = new LinkedHashMap<String, String>(count);
             while (count-- > 0) {
-                String name = StringUtils.readString(in);
-                String childId = StringUtils.readString(in);
+                String name = IOUtils.readString(in);
+                String childId = IOUtils.readString(in);
                 childEntries.put(name, childId);
             }
             return new Node(id, props, childEntries);

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java Tue Aug  9 15:25:56 2011
@@ -23,7 +23,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.jackrabbit.mk.util.StringUtils;
+import org.apache.jackrabbit.mk.util.IOUtils;
 
 /**
  *
@@ -53,7 +53,7 @@ public class PersistentId {
         try {
             if (id == null) {
                 DataInputStream in = new DataInputStream(new FileInputStream(file));
-                id = StringUtils.readString(in);
+                id = IOUtils.readString(in);
                 in.close();
             }
             return id;
@@ -66,7 +66,7 @@ public class PersistentId {
         rwLock.writeLock().lock();
         try {
             DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
-            StringUtils.writeString(out, id);
+            IOUtils.writeString(out, id);
             out.close();
             this.id = id;
         } finally {

Added: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/IOUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/IOUtils.java?rev=1155401&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/IOUtils.java (added)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/IOUtils.java Tue Aug  9 15:25:56 2011
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.util;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Input/output utility methods.
+ */
+public class IOUtils {
+
+    /**
+     * Try to read the given number of bytes to the buffer. This method reads
+     * until the maximum number of bytes have been read or until the end of
+     * file.
+     *
+     * @param in the input stream
+     * @param buffer the output buffer
+     * @param off the offset in the buffer
+     * @param max the number of bytes to read at most
+     * @return the number of bytes read, 0 meaning EOF
+     */
+    public static int readFully(InputStream in, byte[] buffer, int off, int max) throws IOException {
+        int len = Math.min(max, buffer.length);
+        int result = 0;
+        while (len > 0) {
+            int l = in.read(buffer, off, len);
+            if (l < 0) {
+                break;
+            }
+            result += l;
+            off += l;
+            len -= l;
+        }
+        return result;
+    }
+
+    /**
+     * Skip a number of bytes in an input stream.
+     *
+     * @param in the input stream
+     * @param skip the number of bytes to skip
+     * @throws EOFException if the end of file has been reached before all bytes
+     *             could be skipped
+     * @throws IOException if an IO exception occurred while skipping
+     */
+    public static void skipFully(InputStream in, long skip) throws IOException {
+        while (skip > 0) {
+            long skipped = in.skip(skip);
+            if (skipped <= 0) {
+                throw new EOFException();
+            }
+            skip -= skipped;
+        }
+    }
+
+    /**
+     * Write a String. This will first write the length as 4 bytes, and then the
+     * UTF-8 encoded string.
+     *
+     * @param out the data output stream
+     * @param s the string (maximum length about 2 GB)
+     */
+    public static void writeString(OutputStream out, String s) throws IOException {
+        byte[] data = s.getBytes("UTF-8");
+        writeVarInt(out, data.length);
+        out.write(data);
+    }
+
+    /**
+     * Read a String. This will first read the length as 4 bytes, and then the
+     * UTF-8 encoded string.
+     *
+     * @param in the data input stream
+     * @return the string
+     */
+    public static String readString(InputStream in) throws IOException {
+        int len = readVarInt(in);
+        byte[] data = new byte[len];
+        for (int pos = 0; pos < len;) {
+            int l = in.read(data, pos, data.length - pos);
+            if (l < 0) {
+                throw new EOFException();
+            }
+            pos += l;
+        }
+        return new String(data, "UTF-8");
+    }
+
+    /**
+     * Write a variable size integer.
+     * Negative values need 5 bytes.
+     *
+     * @param out the output stream
+     * @param x the value
+     */
+    public static void writeVarInt(OutputStream out, int x) throws IOException {
+        while ((x & ~0x7f) != 0) {
+            out.write((x & 0x7f) | 0x80);
+            x >>>= 7;
+        }
+        out.write(x);
+    }
+
+    /**
+     * Read a variable size integer.
+     *
+     * @param in the input stream
+     * @return the integer
+     */
+    public static int readVarInt(InputStream in) throws IOException {
+        int x = (byte) in.read();
+        if (x >= 0) {
+            return x;
+        }
+        x &= 0x7f;
+        for (int s = 7;; s += 7) {
+            int b = (byte) in.read();
+            x |= (b & 0x7f) << s;
+            if (b >= 0) {
+                return x;
+            }
+        }
+    }
+
+    /**
+     * Write a variable size long.
+     * Negative values need 10 bytes.
+     *
+     * @param out the output stream
+     * @param x the value
+     */
+    public static void writeVarLong(OutputStream out, long x) throws IOException {
+        while ((x & ~0x7f) != 0) {
+            out.write((byte) ((x & 0x7f) | 0x80));
+            x >>>= 7;
+        }
+        out.write((byte) x);
+    }
+
+    /**
+     * Read a variable size long.
+     *
+     * @param in the input stream
+     * @return the long
+     */
+    public static long readVarLong(InputStream in) throws IOException {
+        long x = (byte) in.read();
+        if (x >= 0) {
+            return x;
+        }
+        x &= 0x7f;
+        for (int s = 7;; s += 7) {
+            long b = (byte) in.read();
+            x |= (b & 0x7f) << s;
+            if (b >= 0) {
+                return x;
+            }
+        }
+    }
+
+}

Added: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/SmallLRUCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/SmallLRUCache.java?rev=1155401&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/SmallLRUCache.java (added)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/SmallLRUCache.java Tue Aug  9 15:25:56 2011
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * This class implements a small LRU object cache.
+ *
+ * @param <K> the key
+ * @param <V> the value
+ */
+public class SmallLRUCache<K, V> extends LinkedHashMap<K, V> {
+
+    private static final long serialVersionUID = 1L;
+    private int size;
+
+    private SmallLRUCache(int size) {
+        super(size, (float) 0.75, true);
+        this.size = size;
+    }
+
+    /**
+     * Create a new object with all elements of the given collection.
+     *
+     * @param <K> the key type
+     * @param <V> the value type
+     * @param size the number of elements
+     * @return the object
+     */
+    public static <K, V> SmallLRUCache<K, V> newInstance(int size) {
+        return new SmallLRUCache<K, V>(size);
+    }
+
+    public void setMaxSize(int size) {
+        this.size = size;
+    }
+
+    protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+        return size() > size;
+    }
+
+}

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java Tue Aug  9 15:25:56 2011
@@ -16,11 +16,6 @@
  */
 package org.apache.jackrabbit.mk.util;
 
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
 /**
  * Some string utility methods.
  */
@@ -96,73 +91,4 @@ public class StringUtils {
         }
     }
 
-    /**
-     * Write a String. This will first write the length as 4 bytes, and then the
-     * UTF-8 encoded string.
-     *
-     * @param out the data output stream
-     * @param s the string (maximum length about 2 GB)
-     */
-    public static void writeString(OutputStream out, String s) throws IOException {
-        byte[] data = s.getBytes("UTF-8");
-        writeVarInt(out, data.length);
-        out.write(data);
-    }
-
-    /**
-     * Read a String. This will first read the length as 4 bytes, and then the
-     * UTF-8 encoded string.
-     *
-     * @param in the data input stream
-     * @return the string
-     */
-    public static String readString(InputStream in) throws IOException {
-        int len = readVarInt(in);
-        byte[] data = new byte[len];
-        for (int pos = 0; pos < len;) {
-            int l = in.read(data, pos, data.length - pos);
-            if (l < 0) {
-                throw new EOFException();
-            }
-            pos += l;
-        }
-        return new String(data, "UTF-8");
-    }
-
-    /**
-     * Write a variable size integer.
-     * Negative values need 5 bytes.
-     *
-     * @param out the output stream
-     * @param x the value
-     */
-    public static void writeVarInt(OutputStream out, int x) throws IOException {
-        while ((x & ~0x7f) != 0) {
-            out.write((x & 0x7f) | 0x80);
-            x >>>= 7;
-        }
-        out.write(x);
-    }
-
-    /**
-     * Read a variable size integer.
-     *
-     * @param in the input stream
-     * @return the integer
-     */
-    public static int readVarInt(InputStream in) throws IOException {
-        int x = (byte) in.read();
-        if (x >= 0) {
-            return x;
-        }
-        x &= 0x7f;
-        for (int s = 7;; s += 7) {
-            int b = (byte) in.read();
-            x |= (b & 0x7f) << s;
-            if (b >= 0) {
-                return x;
-            }
-        }
-    }
-
 }

Modified: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DataStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DataStoreTest.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DataStoreTest.java (original)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DataStoreTest.java Tue Aug  9 15:25:56 2011
@@ -30,8 +30,8 @@ import org.apache.jackrabbit.mk.util.Mic
  */
 public class DataStoreTest extends TestCase {
 
-    // private static final String URL = "fs:{homeDir}/target;clean";
-    private static final String URL = "mem:";
+    private static final String URL = "fs:{homeDir}/target;clean";
+    // private static final String URL = "mem:";
 
     private MicroKernel mk;
 
@@ -52,28 +52,29 @@ public class DataStoreTest extends TestC
     }
 
     public void testLarge() throws IOException {
-        doTest(100000, 1);
+        doTest(1000000, 1);
     }
 
     private void doTest(int maxLength, int count) throws IOException {
         String[] s = new String[count * 2];
         Random r = new Random(0);
         for (int i = 0; i < s.length;) {
-            byte[] data = new byte[r.nextInt(maxLength)];
+            int len = count == 1 ? maxLength : r.nextInt(maxLength);
+            byte[] data = new byte[len];
             r.nextBytes(data);
             s[i++] = mk.write(new ByteArrayInputStream(data));
             s[i++] = mk.write(new ByteArrayInputStream(data));
         }
         r.setSeed(0);
         for (int i = 0; i < s.length;) {
-            int expectedLen = r.nextInt(maxLength);
-            byte[] expectedData = new byte[expectedLen];
+            int len = count == 1 ? maxLength : r.nextInt(maxLength);
+            byte[] expectedData = new byte[len];
             r.nextBytes(expectedData);
-            assertEquals(expectedLen, mk.getLength(s[i++]));
+            assertEquals(len, mk.getLength(s[i++]));
 
             String id = s[i++];
-            doTestReadFully(expectedData, expectedLen, id);
-            doTestRead(expectedData, expectedLen, id);
+            doTestReadFully(expectedData, len, id);
+            doTestRead(expectedData, len, id);
         }
     }
 
@@ -83,12 +84,11 @@ public class DataStoreTest extends TestC
     }
 
     private void assertByteArrayEquals(byte[] expected, int expectedLen, byte[] got) {
-        if (expectedLen != got.length) {
-            System.out.println("?");
-        }
         assertEquals(expectedLen, got.length);
         for (int j = 0; j < expectedLen; j++) {
-            assertEquals(expected[j], got[j]);
+            if (expected[j] != got[j]) {
+                assertEquals("j:" + j, expected[j], got[j]);
+            }
         }
     }
 
@@ -96,6 +96,10 @@ public class DataStoreTest extends TestC
         InputStream in = new MicroKernelInputStream(mk, id);
         Random r = new Random(1);
         ByteArrayOutputStream buff = new ByteArrayOutputStream();
+        int minLen = 0;
+        if (expectedLen > 1000000) {
+            minLen = 4000;
+        }
         while (true) {
             int op = r.nextInt(3);
             if (op == 0) {
@@ -105,7 +109,7 @@ public class DataStoreTest extends TestC
                 }
                 buff.write(x);
             } else if (op == 1) {
-                byte[] x = new byte[r.nextInt(100)];
+                byte[] x = new byte[minLen + r.nextInt(100)];
                 int l = in.read(x);
                 if (l < 0) {
                     break;
@@ -113,7 +117,7 @@ public class DataStoreTest extends TestC
                 buff.write(x, 0, l);
             } else {
                 int offset = r.nextInt(10);
-                int len = r.nextInt(20);
+                int len = minLen + r.nextInt(20);
                 byte[] x = new byte[offset + len];
                 int l = in.read(x, offset, len);
                 if (l < 0) {

Added: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DbStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DbStoreTest.java?rev=1155401&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DbStoreTest.java (added)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/DbStoreTest.java Tue Aug  9 15:25:56 2011
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk;
+
+import java.io.ByteArrayInputStream;
+import java.sql.Connection;
+import java.util.Random;
+import junit.framework.TestCase;
+import org.apache.jackrabbit.mk.datastore.DbStore;
+import org.h2.jdbcx.JdbcConnectionPool;
+
+/**
+ * Tests the DbStore implementation.
+ */
+public class DbStoreTest extends TestCase {
+
+    private DbStore store;
+    private Connection sentinel;
+
+    public void setUp() throws Exception {
+        Class.forName("org.h2.Driver");
+        JdbcConnectionPool cp = JdbcConnectionPool.create("jdbc:h2:mem:", "", "");
+        sentinel = cp.getConnection();
+        store = new DbStore();
+        store.setConnectionPool(cp);
+        store.setBlockSize(128);
+        store.setBlockSizeMin(32);
+    }
+
+    public void tearDown() throws Exception {
+        sentinel.close();
+    }
+
+    public void testSmall() throws Exception {
+        doTest(10, 1000);
+    }
+
+    public void testMedium() throws Exception {
+        doTest(100, 100);
+    }
+
+    public void testLarge() throws Exception {
+        doTest(1000, 10);
+    }
+
+    private void doTest(int maxLength, int count) throws Exception {
+        String[] s = new String[count * 2];
+        Random r = new Random(0);
+        for (int i = 0; i < s.length;) {
+            byte[] data = new byte[r.nextInt(maxLength)];
+            r.nextBytes(data);
+            s[i++] = store.writeBlob(new ByteArrayInputStream(data));
+            s[i++] = store.writeBlob(new ByteArrayInputStream(data));
+        }
+        r.setSeed(0);
+        for (int i = 0; i < s.length;) {
+            int expectedLen = r.nextInt(maxLength);
+            byte[] expectedData = new byte[expectedLen];
+            r.nextBytes(expectedData);
+            if (expectedLen != store.getBlobLength(s[i])) {
+                System.out.println("?");
+                assertEquals(expectedLen, store.getBlobLength(s[i++]));
+            }
+            assertEquals(expectedLen, store.getBlobLength(s[i++]));
+
+            String id = s[i++];
+            doTestRead(expectedData, expectedLen, id);
+        }
+    }
+
+    private void assertByteArrayEquals(byte[] expected, int expectedLen, byte[] got) {
+        assertEquals(expectedLen, got.length);
+        for (int j = 0; j < expectedLen; j++) {
+            if (expected[j] != got[j]) {
+                assertEquals("j:" + j, expected[j], got[j]);
+            }
+        }
+    }
+
+    private void doTestRead(byte[] expectedData, int expectedLen, String id) throws Exception {
+        byte[] got = readFully(id);
+        assertByteArrayEquals(expectedData, expectedLen, got);
+    }
+
+    public byte[] readFully(String id) throws Exception {
+        int len = (int) store.getBlobLength(id);
+        byte[] buff = new byte[len];
+        int off = 0, remaining = len;
+        while (remaining > 0) {
+            int l = store.readBlob(id, off, buff, off, remaining);
+            off += l;
+            remaining -= l;
+        }
+        return buff;
+    }
+
+}

Modified: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java?rev=1155401&r1=1155400&r2=1155401&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java (original)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java Tue Aug  9 15:25:56 2011
@@ -36,7 +36,7 @@ public class UtilsTest extends TestCase 
                 s += p;
             }
             ByteArrayOutputStream out = new ByteArrayOutputStream();
-            StringUtils.writeString(out, s);
+            IOUtils.writeString(out, s);
             byte[] data = out.toByteArray();
             ByteArrayInputStream in = new ByteArrayInputStream(data) {
                 public int read(byte[] b, int off, int len) {
@@ -46,47 +46,95 @@ public class UtilsTest extends TestCase 
                     return super.read(b, off, len);
                 }
             };
-            String t = StringUtils.readString(in);
+            String t = IOUtils.readString(in);
             assertEquals(s, t);
             assertEquals(-1, in.read());
         }
     }
 
-    public void testVariableSizeInt() throws IOException {
-        testVar(0, 1);
-        testVar(0x7f, 1);
-        testVar(0x80, 2);
-        testVar(0x3fff, 2);
-        testVar(0x4000, 3);
-        testVar(0x1fffff, 3);
-        testVar(0x200000, 4);
-        testVar(0xfffffff, 4);
-        testVar(0x10000000, 5);
-        testVar(-1, 5);
+    public void testVarInt() throws IOException {
+        testVarInt(0, 1);
+        testVarInt(0x7f, 1);
+        testVarInt(0x80, 2);
+        testVarInt(0x3fff, 2);
+        testVarInt(0x4000, 3);
+        testVarInt(0x1fffff, 3);
+        testVarInt(0x200000, 4);
+        testVarInt(0xfffffff, 4);
+        testVarInt(0x10000000, 5);
+        testVarInt(-1, 5);
         for (int x = 0; x < 0x20000; x++) {
-            testVar(x, 0);
-            testVar(Integer.MIN_VALUE + x, 0);
-            testVar(Integer.MAX_VALUE - x, 5);
-            testVar(0x200000 + x - 100, 0);
-            testVar(0x10000000 + x - 100, 0);
+            testVarInt(x, 0);
+            testVarInt(Integer.MIN_VALUE + x, 0);
+            testVarInt(Integer.MAX_VALUE - x, 5);
+            testVarInt(0x200000 + x - 100, 0);
+            testVarInt(0x10000000 + x - 100, 0);
         }
         Random r = new Random(1);
         for (int i=0; i < 100000; i++) {
-            testVar(r.nextInt(), 0);
-            testVar(r.nextInt(10000000), 0);
+            testVarInt(r.nextInt(), 0);
+            testVarInt(r.nextInt(10000000), 0);
         }
     }
 
-    private void testVar(int x, int expectedLen) throws IOException {
+    public void testVarLong() throws IOException {
+        testVarLong(0, 1);
+        testVarLong(0x7f, 1);
+        testVarLong(0x80, 2);
+        testVarLong(0x3fff, 2);
+        testVarLong(0x4000, 3);
+        testVarLong(0x1fffff, 3);
+        testVarLong(0x200000, 4);
+        testVarLong(0xfffffff, 4);
+        testVarLong(0x10000000, 5);
+        testVarLong(0x1fffffffL, 5);
+        testVarLong(0x2000000000L, 6);
+        testVarLong(0x3ffffffffffL, 6);
+        testVarLong(0x40000000000L, 7);
+        testVarLong(0x1ffffffffffffL, 7);
+        testVarLong(0x2000000000000L, 8);
+        testVarLong(0xffffffffffffffL, 8);
+        testVarLong(0x100000000000000L, 9);
+        testVarLong(-1, 10);
+        for (int x = 0; x < 0x20000; x++) {
+            testVarLong(x, 0);
+            testVarLong(Long.MIN_VALUE + x, 0);
+            testVarLong(Long.MAX_VALUE - x, 9);
+            testVarLong(0x200000 + x - 100, 0);
+            testVarLong(0x10000000 + x - 100, 0);
+        }
+        Random r = new Random(1);
+        for (int i=0; i < 100000; i++) {
+            testVarLong(r.nextLong(), 0);
+            testVarLong(r.nextInt(Integer.MAX_VALUE), 0);
+        }
+    }
+
+
+    private void testVarInt(int x, int expectedLen) throws IOException {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
-        StringUtils.writeVarInt(out, x);
+        IOUtils.writeVarInt(out, x);
         byte[] data = out.toByteArray();
         assertTrue(data.length <= 5);
         if (expectedLen > 0) {
             assertEquals(expectedLen, data.length);
         }
         ByteArrayInputStream in = new ByteArrayInputStream(data);
-        int x2 = StringUtils.readVarInt(in);
+        int x2 = IOUtils.readVarInt(in);
+        assertEquals(x, x2);
+        assertEquals(-1, in.read());
+    }
+
+    private void testVarLong(long x, int expectedLen) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        IOUtils.writeVarLong(out, x);
+        byte[] data = out.toByteArray();
+        assertTrue(data.length <= 10);
+        if (expectedLen > 0) {
+            assertEquals(expectedLen, data.length);
+        }
+        ByteArrayInputStream in = new ByteArrayInputStream(data);
+        long x2 = IOUtils.readVarLong(in);
         assertEquals(x, x2);
         assertEquals(-1, in.read());
     }



Mime
View raw message