jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1458181 - in /jackrabbit/oak/trunk/oak-mongomk/src: main/java/org/apache/jackrabbit/mongomk/prototype/ test/java/org/apache/jackrabbit/mongomk/prototype/
Date Tue, 19 Mar 2013 09:03:31 GMT
Author: thomasm
Date: Tue Mar 19 09:03:31 2013
New Revision: 1458181

URL: http://svn.apache.org/r1458181
Log:
OAK-619 Lock-free MongoMK implementation (fixes child list caching)

Modified:
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
Tue Mar 19 09:03:31 2013
@@ -106,8 +106,14 @@ public class Commit {
         addedNodes.add(n.path);
     }
 
-    boolean isEmpty() {
-        return operations.isEmpty();
+    /**
+     * Apply the changes to the document store and the cache.
+     */
+    void apply() {
+        if (!operations.isEmpty()) {
+            applyToDocumentStore();
+            applyToCache();
+        }
     }
 
     /**

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
Tue Mar 19 09:03:31 2013
@@ -26,8 +26,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import javax.annotation.Nonnull;
@@ -263,37 +261,39 @@ public class MongoMK implements MicroKer
         return x.compareRevisionTime(previous) > 0;
     }
     
-    public Node.Children readChildren(final String path, final String nodeId,
-                                      final Revision rev, final int limit) {
-        try {
-            return nodeChildrenCache.get(nodeId, new Callable<Children>() {
-                @Override
-                public Children call() throws Exception {
-                    String from = PathUtils.concat(path, "a");
-                    from = Utils.getIdFromPath(from);
-                    from = from.substring(0, from.length() - 1);
-                    String to = PathUtils.concat(path, "z");
-                    to = Utils.getIdFromPath(to);
-                    to = to.substring(0, to.length() - 2) + "0";
-                    List<Map<String, Object>> list = store.query(DocumentStore.Collection.NODES,
from, to, limit);
-                    Children c = new Children(path, nodeId, rev);
-                    for (Map<String, Object> e : list) {
-                        // filter out deleted children
-                        if (getLiveRevision(e, rev) == null) {
-                            continue;
-                        }
-                        // TODO put the whole node in the cache
-                        String id = e.get(UpdateOp.ID).toString();
-                        String p = Utils.getPathFromId(id);
-                        c.children.add(p);
-                    }
-                    return c;
-                }
-            });
-        } catch (ExecutionException e) {
-            throw new IllegalStateException("Error occurred while fetching node children
for node "+nodeId,e);
+    public Children getChildren(String path, Revision rev, int limit) {
+        checkRevisionAge(rev, path);
+        String key = path + "@" + rev;
+        Children children = nodeChildrenCache.getIfPresent(key);
+        if (children == null) {
+            children = readChildren(path, rev, limit);
+            if (children != null) {
+                nodeChildrenCache.put(key, children);
+            }
+        }
+        return children;        
+    }
+    
+    Node.Children readChildren(String path, Revision rev, int limit) {
+        String from = PathUtils.concat(path, "a");
+        from = Utils.getIdFromPath(from);
+        from = from.substring(0, from.length() - 1);
+        String to = PathUtils.concat(path, "z");
+        to = Utils.getIdFromPath(to);
+        to = to.substring(0, to.length() - 2) + "0";
+        List<Map<String, Object>> list = store.query(DocumentStore.Collection.NODES,
from, to, limit);
+        Children c = new Children(path, rev);
+        for (Map<String, Object> e : list) {
+            // filter out deleted children
+            if (getLiveRevision(e, rev) == null) {
+                continue;
+            }
+            // TODO put the whole node in the cache
+            String id = e.get(UpdateOp.ID).toString();
+            String p = Utils.getPathFromId(id);
+            c.children.add(p);
         }
-
+        return c;
     }
 
     private Node readNode(String path, Revision rev) {
@@ -429,17 +429,18 @@ public class MongoMK implements MicroKer
         Revision toRev = Revision.fromString(toRevisionId);
         // TODO this does not work well for large child node lists 
         // use a MongoDB index instead
-        Children fromChildren = readChildren(path, from.getId(), fromRev, Integer.MAX_VALUE);
-        Children toChildren = readChildren(path, to.getId(), toRev, Integer.MAX_VALUE);
+        Children fromChildren = getChildren(path, fromRev, Integer.MAX_VALUE);
+        Children toChildren = getChildren(path, toRev, Integer.MAX_VALUE);
         Set<String> childrenSet = new HashSet<String>(toChildren.children);
         for (String n : fromChildren.children) {
             if (!childrenSet.contains(n)) {
                 w.tag('-').key(n).object().endObject().newline();
             } else {
-                // TODO currently all children seem to diff, 
-                // which is not necessarily the case
-                // (compare write counters)
-                w.tag('^').key(n).object().endObject().newline();
+                Node n1 = getNode(n, fromRev);
+                Node n2 = getNode(n, toRev);
+                if (!n1.equals(n2)) {
+                    w.tag('^').key(n).object().endObject().newline();
+                }
             }
         }
         childrenSet = new HashSet<String>(fromChildren.children);
@@ -491,7 +492,7 @@ public class MongoMK implements MicroKer
         json.object();
         n.append(json, includeId);
         // FIXME: must not read all children!
-        Children c = readChildren(path, n.getId(), rev, Integer.MAX_VALUE);
+        Children c = getChildren(path, rev, Integer.MAX_VALUE);
         for (long i = offset; i < c.children.size(); i++) {
             if (maxChildNodes-- <= 0) {
                 break;
@@ -577,10 +578,11 @@ public class MongoMK implements MicroKer
         }
         if (revisionId.startsWith("b")) {
             // just commit to head currently
-            applyCommit(commit);
+            commit.apply();
             // remember branch commit
             branchCommits.put(rev.toString(), revisionId.substring(1));
 
+            headRevision = commit.getRevision();
             return "b" + rev.toString();
 
             // String jsonBranch = branchCommits.remove(revisionId);
@@ -589,7 +591,8 @@ public class MongoMK implements MicroKer
             // branchCommits.put(branchRev, jsonBranch);
             // return branchRev;
         }
-        applyCommit(commit);
+        commit.apply();
+        headRevision = commit.getRevision();
         return rev.toString();
     }
 
@@ -628,8 +631,7 @@ public class MongoMK implements MicroKer
         if (move) {
             markAsDeleted(sourcePath, commit, false);
         }
-        Node.Children c = readChildren(sourcePath, n.getId(),
-                baseRev, Integer.MAX_VALUE);
+        Node.Children c = getChildren(sourcePath, baseRev, Integer.MAX_VALUE);
         for (String srcChildPath : c.children) {
             String childName = PathUtils.getName(srcChildPath);
             String destChildPath = PathUtils.concat(targetPath, childName);
@@ -651,7 +653,7 @@ public class MongoMK implements MicroKer
             nodeCache.invalidate(path + "@" + rev);
             
             if (n != null) {
-                Node.Children c = readChildren(path, n.getId(), rev,
+                Node.Children c = getChildren(path, rev,
                         Integer.MAX_VALUE);
                 for (String childPath : c.children) {
                     markAsDeleted(childPath, commit, true);
@@ -703,15 +705,6 @@ public class MongoMK implements MicroKer
         }
         return revisionId;
     }
-
-    private void applyCommit(Commit commit) {
-        headRevision = commit.getRevision();
-        if (commit.isEmpty()) {
-            return;
-        }
-        commit.applyToDocumentStore();
-        commit.applyToCache();
-    }
     
     public static void parseAddNode(Commit commit, JsopReader t, String path) {
         Node n = new Node(path, commit.getRevision());
@@ -849,8 +842,8 @@ public class MongoMK implements MicroKer
         long newWriteCount = oldWriteCount + writeCountInc;
         Children c = nodeChildrenCache.getIfPresent(path + "@" + (newWriteCount - 1));
         if (isNew || (!isDelete && c != null)) {
-            String id = path + "@" + newWriteCount;
-            Children c2 = new Children(path, id, rev);
+            String key = path + "@" + rev;
+            Children c2 = new Children(path, rev);
             TreeSet<String> set = new TreeSet<String>();
             if (c != null) {
                 set.addAll(c.children);
@@ -858,14 +851,7 @@ public class MongoMK implements MicroKer
             set.removeAll(removed);
             set.addAll(added);
             c2.children.addAll(set);
-            if (nodeChildrenCache.getIfPresent(id) != null) {
-                // TODO should not happend, 
-                // probably a problem with add/delete/add
-                MicroKernelException e = new MicroKernelException(
-                        "New child list already cached for id " + id);
-                LOG.error("Error updating the cache", e);
-            }
-            nodeChildrenCache.put(id, c2);
+            nodeChildrenCache.put(key, c2);
         }
     }
 

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
Tue Mar 19 09:03:31 2013
@@ -95,20 +95,40 @@ public class Node {
         }
     }
     
+    public void setWriteCount(long writeCount) {
+        this.writeCount = writeCount;
+    }
+
+    public long getWriteCount() {
+        return writeCount;
+    }
+    
+    public int hashCode() {
+        return (int) writeCount ^ properties.size() ^ rev.hashCode();
+    }
+    
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        } else if (!(obj instanceof Node)) {
+            return false;
+        }
+        Node other = (Node) obj;
+        return writeCount == other.writeCount;
+    }
+
     /**
      * A list of children for a node.
      */
     static class Children {
         
         final String path;
-        final String id;
         final Revision rev;
         
         final ArrayList<String> children = new ArrayList<String>();
         
-        Children(String path, String id, Revision rev) {
+        Children(String path, Revision rev) {
             this.path = path;
-            this.id = id;
             this.rev = rev;
         }
         
@@ -118,8 +138,4 @@ public class Node {
         
     }
 
-    public void setWriteCount(long writeCount) {
-        this.writeCount = writeCount;
-    }
-
 }

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
Tue Mar 19 09:03:31 2013
@@ -24,11 +24,36 @@ import java.util.TreeMap;
  */
 public class UpdateOp {
     
+    /**
+     * The node id, which contains the depth of the path
+     * (0 for root, 1 for children of the root), and then the path.
+     */
     static final String ID = "_id";
+    
+    /**
+     * The number of write operations to this node.
+     */
     static final String WRITE_COUNT = "_writeCount";
+    
+    /**
+     * The list of recent revisions against this node, where this node is the
+     * root of the commit.
+     */
     static final String REVISIONS = "_revisions";
+    
+    /**
+     * The number of previous documents (documents that contain old revisions of
+     * this node). This property is only set if multiple documents per node
+     * exist. This is the case when a node is updated very often in a short
+     * time, such that the document gets very big.
+     */
     static final String PREVIOUS = "_prev";
+    
+    /**
+     * Whether this node is
+     */
     static final String DELETED = "_deleted";
+    static final String MODIFIED = "_modified";
     
     final String path;
     

Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
Tue Mar 19 09:03:31 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTru
 
 import org.apache.jackrabbit.mongomk.prototype.DocumentStore.Collection;
 import org.apache.jackrabbit.mongomk.prototype.Node.Children;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Lists;
@@ -89,21 +90,35 @@ public class SimpleTest {
     }
     
     @Test
+    @Ignore
     public void diff() {
         MongoMK mk = createMK();
+        
         String rev0 = mk.getHeadRevision();
-        // TODO
-//        String rev1 = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null);
-//        String rev2 = mk.commit("/", "-\"test\"", null, null);
-//        String rev3 = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null);
-//        String test0 = mk.getNodes("/test", rev0, 0, 0, Integer.MAX_VALUE, null);
-//        assertNull(null, test0);
-//        String test1 = mk.getNodes("/test", rev1, 0, 0, Integer.MAX_VALUE, null);
-//        assertEquals("{\"name\":\"Hello\",\":childNodeCount\":0}", test1);
-//        String test2 = mk.getNodes("/test", rev2, 0, 0, Integer.MAX_VALUE, null);
-//        assertNull(null, test2);
-//        String test3 = mk.getNodes("/test", rev3, 0, 0, Integer.MAX_VALUE, null);
-//        assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test3);
+        String rev1 = mk.commit("/", "+\"t1\":{}", null, null);
+        String rev2 = mk.commit("/", "+\"t2\":{}", null, null);
+        String rev3 = mk.commit("/", "+\"t3\":{}", null, null);
+        String rev4 = mk.commit("/", "^\"t2/x\":1", null, null);
+        
+        String r0 = mk.getNodes("/", rev0, 0, 0, Integer.MAX_VALUE, null);
+        assertEquals("{\":childNodeCount\":0}", r0);
+        String r1 = mk.getNodes("/", rev1, 0, 0, Integer.MAX_VALUE, null);
+        assertEquals("{\"t1\":{},\":childNodeCount\":1}", r1);
+        String r2 = mk.getNodes("/", rev2, 0, 0, Integer.MAX_VALUE, null);
+        assertEquals("{\"t1\":{},\"t2\":{},\":childNodeCount\":2}", r2);
+        String r3 = mk.getNodes("/", rev3, 0, 0, Integer.MAX_VALUE, null);
+        assertEquals("{\"t1\":{},\"t2\":{},\"t3\":{},\":childNodeCount\":3}", r3);
+        
+        String diff01 = mk.diff(rev0, rev1, "/", 0).trim();
+        assertEquals("+\"/t1\":{}", diff01);
+        String diff12 = mk.diff(rev1, rev2, "/", 0).trim();
+        assertEquals("+\"/t2\":{}", diff12);
+        String diff23 = mk.diff(rev2, rev3, "/", 0).trim();
+        assertEquals("+\"/t3\":{}", diff23);
+        String diff13 = mk.diff(rev1, rev3, "/", 0).trim();
+        assertEquals("+\"/t2\":{}\n+\"/t3\":{}", diff13);
+        String diff34 = mk.diff(rev3, rev4, "/", 0).trim();
+        assertEquals("^\"/t2\":{}", diff34);
         mk.dispose();
     }
 
@@ -178,15 +193,13 @@ public class SimpleTest {
         String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
         assertEquals("{\"name\":\"Hello\",\":childNodeCount\":0}", test);
         
-        rev = mk.commit("/test", "+\"a\":{\"name\": \"World\"}", null, null);
-        rev = mk.commit("/test", "+\"b\":{\"name\": \"!\"}", null, null);
-        test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
+        String r0 = mk.commit("/test", "+\"a\":{\"name\": \"World\"}", null, null);
+        String r1 = mk.commit("/test", "+\"b\":{\"name\": \"!\"}", null, null);
+        test = mk.getNodes("/test", r0, 0, 0, Integer.MAX_VALUE, null);
         Children c;
-        c = mk.readChildren("/", "1",
-                Revision.fromString(rev), Integer.MAX_VALUE);
+        c = mk.getChildren("/", Revision.fromString(r0), Integer.MAX_VALUE);
         assertEquals("/: [/test]", c.toString());
-        c = mk.readChildren("/test", "2",
-                Revision.fromString(rev), Integer.MAX_VALUE);
+        c = mk.getChildren("/test", Revision.fromString(r1), Integer.MAX_VALUE);
         assertEquals("/test: [/test/a, /test/b]", c.toString());
 
         rev = mk.commit("", "^\"/test\":1", null, null);
@@ -252,23 +265,22 @@ public class SimpleTest {
     public void testDeletion() {
         MongoMK mk = createMK();
 
-        String rev = mk.commit("/", "+\"testDel\":{\"name\": \"Hello\"}", null,
-                null);
-        rev = mk.commit("/testDel", "+\"a\":{\"name\": \"World\"}", null, null);
-        rev = mk.commit("/testDel", "+\"b\":{\"name\": \"!\"}", null, null);
-        rev = mk.commit("/testDel", "+\"c\":{\"name\": \"!\"}", null, null);
+        mk.commit("/", "+\"testDel\":{\"name\": \"Hello\"}", null, null);
+        mk.commit("/testDel", "+\"a\":{\"name\": \"World\"}", null, null);
+        mk.commit("/testDel", "+\"b\":{\"name\": \"!\"}", null, null);
+        String r1 = mk.commit("/testDel", "+\"c\":{\"name\": \"!\"}", null, null);
 
-        Children c = mk.readChildren("/testDel", "1", Revision.fromString(rev),
+        Children c = mk.getChildren("/testDel", Revision.fromString(r1),
                 Integer.MAX_VALUE);
         assertEquals(3, c.children.size());
 
-        rev = mk.commit("/testDel", "-\"c\"", null, null);
-        c = mk.readChildren("/testDel", "2", Revision.fromString(rev),
+        String r2 = mk.commit("/testDel", "-\"c\"", null, null);
+        c = mk.getChildren("/testDel", Revision.fromString(r2),
                 Integer.MAX_VALUE);
         assertEquals(2, c.children.size());
 
-        rev = mk.commit("/", "-\"testDel\"", null, null);
-        Node n = mk.getNode("/testDel", Revision.fromString(rev));
+        String r3 = mk.commit("/", "-\"testDel\"", null, null);
+        Node n = mk.getNode("/testDel", Revision.fromString(r3));
         assertNull(n);
     }
 



Mime
View raw message