jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1146670 - in /jackrabbit/sandbox/microkernel/src: main/java/org/apache/jackrabbit/mk/store/ main/java/org/apache/jackrabbit/mk/util/ test/java/org/apache/jackrabbit/mk/ test/java/org/apache/jackrabbit/mk/util/
Date Thu, 14 Jul 2011 12:45:13 GMT
Author: thomasm
Date: Thu Jul 14 12:45:13 2011
New Revision: 1146670

URL: http://svn.apache.org/viewvc?rev=1146670&view=rev
Log:
Unlimited length values (and node names, property names; and uses less disk space).

Added:
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/LargeObjectTest.java
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java
Modified:
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Commit.java
Thu Jul 14 12:45:13 2011
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.mk.store;
 
 import java.io.DataInputStream;
 import java.io.InputStream;
+import org.apache.jackrabbit.mk.util.StringUtils;
 
 /**
  * Immutable Commit object...
@@ -69,10 +70,10 @@ public class Commit extends StoredObject
             if (in.read() != Constants.COMMIT) {
                 throw new Exception("unknown data format");
             }
-            String rootNodeId = in.readUTF();
+            String rootNodeId = StringUtils.readString(in);
             long commitTS = in.readLong();
-            String msg = in.readUTF();
-            String parentId = in.readUTF();
+            String msg = StringUtils.readString(in);
+            String parentId = StringUtils.readString(in);
             return new Commit(id, rootNodeId, commitTS, "".equals(msg) ? null : msg, "".equals(parentId)
? null : parentId);
         } finally {
             in.close();

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableCommit.java
Thu Jul 14 12:45:13 2011
@@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import org.apache.jackrabbit.mk.util.StringUtils;
 
 /**
  *
@@ -81,10 +82,10 @@ public class MutableCommit {
         DataOutputStream out = new DataOutputStream(baos);
         try {
             out.write(Constants.COMMIT);
-            out.writeUTF(rootNodeId);
+            StringUtils.writeString(out, rootNodeId);
             out.writeLong(commitTS);
-            out.writeUTF(msg == null ? "" : msg);
-            out.writeUTF(parentId == null ? "" : parentId);
+            StringUtils.writeString(out, msg == null ? "" : msg);
+            StringUtils.writeString(out, parentId == null ? "" : parentId);
             out.close();
         } catch (IOException e) {
             throw new RuntimeException(e);

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/MutableNode.java
Thu Jul 14 12:45:13 2011
@@ -24,6 +24,7 @@ import java.io.InputStream;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import org.apache.jackrabbit.mk.util.StringUtils;
 
 /**
  *
@@ -62,25 +63,24 @@ public class MutableNode {
     //--------------------------------------------------------< serialization >
 
     public InputStream toBytes() {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        DataOutputStream out = new DataOutputStream(baos);
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
         try {
             out.write(Constants.NODE);
-            out.writeInt(properties.size());
+            StringUtils.writeVarInt(out, properties.size());
             for (Map.Entry<String, String> entry : properties.entrySet()) {
-                out.writeUTF(entry.getKey());
-                out.writeUTF(entry.getValue());
+                StringUtils.writeString(out, entry.getKey());
+                StringUtils.writeString(out, entry.getValue());
             }
-            out.writeInt(childEntries.size());
+            StringUtils.writeVarInt(out, childEntries.size());
             for (Map.Entry<String, String> entry : childEntries.entrySet()) {
-                out.writeUTF(entry.getKey());
-                out.writeUTF(entry.getValue());
+                StringUtils.writeString(out, entry.getKey());
+                StringUtils.writeString(out, entry.getValue());
             }
             out.close();
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
 
-        return new ByteArrayInputStream(baos.toByteArray());
+        return new ByteArrayInputStream(out.toByteArray());
     }
 }

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/Node.java
Thu Jul 14 12:45:13 2011
@@ -22,6 +22,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import org.apache.jackrabbit.mk.util.StringUtils;
 
 /**
  * Immutable Node object...
@@ -70,18 +71,18 @@ public class Node extends StoredObject {
             if (in.read() != Constants.NODE) {
                 throw new Exception("unknown data format");
             }
-            int count = in.readInt();
+            int count = StringUtils.readVarInt(in);
             Map<String, String> props = new HashMap<String, String>(count);
             while (count-- > 0) {
-                String name = in.readUTF();
-                String value = in.readUTF();
+                String name = StringUtils.readString(in);
+                String value = StringUtils.readString(in);
                 props.put(name, value);
             }
-            count = in.readInt();
+            count = StringUtils.readVarInt(in);
             LinkedHashMap<String, String> childEntries = new LinkedHashMap<String,
String>(count);
             while (count-- > 0) {
-                String name = in.readUTF();
-                String childId = in.readUTF();
+                String name = StringUtils.readString(in);
+                String childId = StringUtils.readString(in);
                 childEntries.put(name, childId);
             }
             return new Node(id, props, childEntries);

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/store/PersistentId.java
Thu Jul 14 12:45:13 2011
@@ -23,6 +23,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.jackrabbit.mk.util.StringUtils;
 
 /**
  *
@@ -52,7 +53,7 @@ public class PersistentId {
         try {
             if (id == null) {
                 DataInputStream in = new DataInputStream(new FileInputStream(file));
-                id = in.readUTF();
+                id = StringUtils.readString(in);
                 in.close();
             }
             return id;
@@ -65,7 +66,7 @@ public class PersistentId {
         rwLock.writeLock().lock();
         try {
             DataOutputStream out = new DataOutputStream(new FileOutputStream(file));
-            out.writeUTF(id);
+            StringUtils.writeString(out, id);
             out.close();
             this.id = id;
         } finally {

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java?rev=1146670&r1=1146669&r2=1146670&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/StringUtils.java
Thu Jul 14 12:45:13 2011
@@ -16,6 +16,11 @@
  */
 package org.apache.jackrabbit.mk.util;
 
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
 /**
  * Some string utility methods.
  */
@@ -91,4 +96,73 @@ public class StringUtils {
         }
     }
 
+    /**
+     * Write a String. This will first write the length as 4 bytes, and then the
+     * UTF-8 encoded string.
+     *
+     * @param out the data output stream
+     * @param s the string (maximum length about 2 GB)
+     */
+    public static void writeString(OutputStream out, String s) throws IOException {
+        byte[] data = s.getBytes("UTF-8");
+        writeVarInt(out, data.length);
+        out.write(data);
+    }
+
+    /**
+     * Read a String. This will first read the length as 4 bytes, and then the
+     * UTF-8 encoded string.
+     *
+     * @param in the data input stream
+     * @return the string
+     */
+    public static String readString(InputStream in) throws IOException {
+        int len = readVarInt(in);
+        byte[] data = new byte[len];
+        for (int pos = 0; pos < len;) {
+            int l = in.read(data, pos, data.length - pos);
+            if (l < 0) {
+                throw new EOFException();
+            }
+            pos += l;
+        }
+        return new String(data, "UTF-8");
+    }
+
+    /**
+     * Write a variable size integer.
+     * Negative values need 5 bytes.
+     *
+     * @param out the output stream
+     * @param x the value
+     */
+    public static void writeVarInt(OutputStream out, int x) throws IOException {
+        while ((x & ~0x7f) != 0) {
+            out.write((x & 0x7f) | 0x80);
+            x >>>= 7;
+        }
+        out.write(x);
+    }
+
+    /**
+     * Read a variable size integer.
+     *
+     * @param in the input stream
+     * @return the integer
+     */
+    public static int readVarInt(InputStream in) throws IOException {
+        int x = (byte) in.read();
+        if (x >= 0) {
+            return x;
+        }
+        x &= 0x7f;
+        for (int s = 7;; s += 7) {
+            int b = (byte) in.read();
+            x |= (b & 0x7f) << s;
+            if (b >= 0) {
+                return x;
+            }
+        }
+    }
+
 }

Added: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/LargeObjectTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/LargeObjectTest.java?rev=1146670&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/LargeObjectTest.java
(added)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/LargeObjectTest.java
Thu Jul 14 12:45:13 2011
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk;
+
+import junit.framework.TestCase;
+import org.apache.jackrabbit.mk.api.MicroKernel;
+
+/**
+ * Test large node, property, and values (longer than 65 KB).
+ */
+public class LargeObjectTest extends TestCase {
+
+    private static final String URL = "fs:{homeDir};clean";
+    // private static final String URL = "mem:";
+
+    private MicroKernel mk;
+
+    public void setUp() throws Exception {
+        mk = MicroKernelFactory.getInstance(URL);
+    }
+
+    public void tearDown() throws InterruptedException {
+        mk.dispose();
+    }
+
+    public void test() {
+
+        String head = mk.getHeadRevision();
+        head = mk.commit("/", "+ \"test\": { \"data\": \"Hello World\" }", head, null);
+
+        String large = new String(new char[100000]).replace((char) 0, 'x');
+        head = mk.commit("/", "^ \"test/data\": \"" + large + "\"", head, null);
+
+        head = mk.commit("/", "+ \"test2\": { \"" + large + "\": \"Hello World\" }", head,
null);
+        head = mk.commit("/", "^ \"test2/" + large + "\": \"" + large + "\"", head, null);
+
+        head = mk.commit("/", "+ \"" + large + "\": { \"" + large + "\": \"Hello World\"
}", head, null);
+        head = mk.commit("/", "^ \"" + large + "/" + large + "\": \"" + large + "\"", head,
null);
+
+    }
+
+}

Added: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java?rev=1146670&view=auto
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java
(added)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/UtilsTest.java
Thu Jul 14 12:45:13 2011
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Random;
+import junit.framework.TestCase;
+
+/**
+ * Test the utility classes.
+ */
+public class UtilsTest extends TestCase {
+
+    public void testStringReadWrite() throws IOException {
+        final Random r = new Random(1);
+        for (int i = 0; i < 100000; i += i / 10 + 1) {
+            String s = "";
+            for (int j = 0; j < 10; j++) {
+                String p = new String(new char[i]).replace((char) 0, 'a');
+                s += p;
+            }
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            StringUtils.writeString(out, s);
+            byte[] data = out.toByteArray();
+            ByteArrayInputStream in = new ByteArrayInputStream(data) {
+                public int read(byte[] b, int off, int len) {
+                    if (r.nextBoolean()) {
+                        len = r.nextInt(len);
+                    }
+                    return super.read(b, off, len);
+                }
+            };
+            String t = StringUtils.readString(in);
+            assertEquals(s, t);
+            assertEquals(-1, in.read());
+        }
+    }
+
+    public void testVariableSizeInt() throws IOException {
+        testVar(0, 1);
+        testVar(0x7f, 1);
+        testVar(0x80, 2);
+        testVar(0x3fff, 2);
+        testVar(0x4000, 3);
+        testVar(0x1fffff, 3);
+        testVar(0x200000, 4);
+        testVar(0xfffffff, 4);
+        testVar(0x10000000, 5);
+        testVar(-1, 5);
+        for (int x = 0; x < 0x20000; x++) {
+            testVar(x, 0);
+            testVar(Integer.MIN_VALUE + x, 0);
+            testVar(Integer.MAX_VALUE - x, 5);
+            testVar(0x200000 + x - 100, 0);
+            testVar(0x10000000 + x - 100, 0);
+        }
+        Random r = new Random(1);
+        for (int i=0; i < 100000; i++) {
+            testVar(r.nextInt(), 0);
+            testVar(r.nextInt(10000000), 0);
+        }
+    }
+
+    private void testVar(int x, int expectedLen) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        StringUtils.writeVarInt(out, x);
+        byte[] data = out.toByteArray();
+        assertTrue(data.length <= 5);
+        if (expectedLen > 0) {
+            assertEquals(expectedLen, data.length);
+        }
+        ByteArrayInputStream in = new ByteArrayInputStream(data);
+        int x2 = StringUtils.readVarInt(in);
+        assertEquals(x, x2);
+        assertEquals(-1, in.read());
+    }
+
+}



Mime
View raw message