jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1243562 - in /jackrabbit/sandbox/microkernel/src: main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java main/java/org/apache/jackrabbit/mk/util/Sync.java test/java/org/apache/jackrabbit/mk/util/SyncTest.java
Date Mon, 13 Feb 2012 15:32:40 GMT
Author: thomasm
Date: Mon Feb 13 15:32:39 2012
New Revision: 1243562

URL: http://svn.apache.org/viewvc?rev=1243562&view=rev
Log:
Support large child node lists.

Modified:
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
    jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java
    jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
(original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/mem/NodeImpl.java
Mon Feb 13 15:32:39 2012
@@ -90,6 +90,7 @@ public class NodeImpl implements Cache.V
     private int memory;
     private long descendantCount;
     private int descendantInlineCount;
+    private boolean hasMoreChildNodes;
 
     private byte[] hash;
 
@@ -449,17 +450,20 @@ public class NodeImpl implements Cache.V
             node.setPath(path);
         }
         if (!t.matches('}')) {
+            long parsedChildCount = 0;
+            int parsedChildren = 0;
             do {
                 String key = t.readString();
                 t.read(':');
                 if (t.matches('{')) {
                     String childPath = path == null || key == null ? null : PathUtils.concat(path,
key);
                     node.addChildNode(key, false, null, parse(map, t, revId, childPath));
+                    parsedChildren++;
                 } else {
                     String value = t.readRawValue().trim();
                     if (key.length() > 0 && key.charAt(0) == ':') {
                         if (key.equals(CHILREN_COUNT)) {
-                            // ignore
+                            parsedChildCount = Long.parseLong(value);
                         } else if (key.equals(HASH)) {
                             value = JsopTokenizer.decodeQuoted(value);
                             node.hash = StringUtils.convertHexToBytes(value);
@@ -474,6 +478,9 @@ public class NodeImpl implements Cache.V
                 }
             } while (t.matches(','));
             t.read('}');
+            if (parsedChildCount > 0 && parsedChildren < parsedChildCount)
{
+                node.hasMoreChildNodes = true;
+            }
         }
         return node;
     }
@@ -716,4 +723,8 @@ public class NodeImpl implements Cache.V
         return false;
     }
 
+    public boolean hasMoreChildNodes() {
+        return hasMoreChildNodes;
+    }
+
 }

Modified: jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java (original)
+++ jackrabbit/sandbox/microkernel/src/main/java/org/apache/jackrabbit/mk/util/Sync.java Mon
Feb 13 15:32:39 2012
@@ -16,6 +16,7 @@
  */
 package org.apache.jackrabbit.mk.util;
 
+import java.util.Iterator;
 import org.apache.jackrabbit.mk.api.MicroKernel;
 import org.apache.jackrabbit.mk.mem.NodeImpl;
 
@@ -31,6 +32,8 @@ public class Sync {
     private MicroKernel sourceMk, targetMk;
     private String sourceRev, targetRev;
     private String sourcePath, targetPath = "/";
+    private boolean useContentHashOptimization;
+    private int childNodesPerBatch = 100;
 
     private Handler handler;
 
@@ -62,6 +65,42 @@ public class Sync {
         targetPath = path;
     }
 
+    /**
+     * Whether to use the content hash optimization if available.
+     *
+     * @return true if the optimization should be used
+     */
+    public boolean getUseContentHashOptimization() {
+        return useContentHashOptimization;
+    }
+
+    /**
+     * Use the content hash optimization if available.
+     *
+     * @param useContentHashOptimization the new value
+     */
+    public void setUseContentHashOptimization(boolean useContentHashOptimization) {
+        this.useContentHashOptimization = useContentHashOptimization;
+    }
+
+    /**
+     * Get the number of child nodes to request in one call.
+     *
+     * @return the number of child nodes to request
+     */
+    public int getChildNodesPerBatch() {
+        return childNodesPerBatch;
+    }
+
+    /**
+     * Set the number of child nodes to request in one call.
+     *
+     * @param childNodesPerBatch the number of child nodes to request
+     */
+    public void setChildNodesPerBatch(int childNodesPerBatch) {
+        this.childNodesPerBatch = childNodesPerBatch;
+    }
+
     public void run(Handler handler) {
         this.handler = handler;
         visit("");
@@ -71,12 +110,11 @@ public class Sync {
         String source = PathUtils.concat(sourcePath, relPath);
         String target = PathUtils.concat(targetPath, relPath);
         NodeImpl s = null, t = null;
-        // TODO support large child node lists
         if (sourceMk.nodeExists(source, sourceRev)) {
-            s = NodeImpl.parse(sourceMk.getNodes(source, sourceRev, 0, 0, Integer.MAX_VALUE));
+            s = NodeImpl.parse(sourceMk.getNodes(source, sourceRev, 0, 0, childNodesPerBatch));
         }
         if (targetMk != null && targetMk.nodeExists(target, targetRev)) {
-            t = NodeImpl.parse(targetMk.getNodes(target, targetRev, 0, 0, Integer.MAX_VALUE));
+            t = NodeImpl.parse(targetMk.getNodes(target, targetRev, 0, 0, childNodesPerBatch));
         }
         if (s == null || t == null) {
             if (s == t) {
@@ -100,34 +138,91 @@ public class Sync {
                 handler.setProperty(target, name, sourceValue);
             }
         }
-        for (int i = 0; t != null && i < t.getPropertyCount(); i++) {
-            String name = t.getProperty(i);
-            // if it exists in the source, it's already updated
-            if (!s.hasProperty(name)) {
-                handler.setProperty(target, name, null);
+        if (t != null) {
+            for (int i = 0; i < t.getPropertyCount(); i++) {
+                String name = t.getProperty(i);
+                // if it exists in the source, it's already updated
+                if (!s.hasProperty(name)) {
+                    handler.setProperty(target, name, null);
+                }
             }
         }
         // child nodes
-        for (int i = 0;; i++) {
-            String name = s.getChildNodeName(i);
-            if (name == null) {
-                break;
-            }
+        Iterator<String> it = s.hasMoreChildNodes() ?
+                getAllChildNodeNames(sourceMk, source, sourceRev, childNodesPerBatch) :
+                s.getChildNodeNames(Integer.MAX_VALUE);
+        while (it.hasNext()) {
+            String name = it.next();
             visit(PathUtils.concat(relPath, name));
         }
-        for (int i = 0; t != null; i++) {
-            String name = t.getChildNodeName(i);
-            if (name == null) {
-                break;
-            }
-            // if it exists in the source, it's already updated
-            if (!s.exists(name)) {
-                visit(PathUtils.concat(relPath, name));
+        if (t != null) {
+            it = t.hasMoreChildNodes() ?
+                    getAllChildNodeNames(targetMk, target, targetRev, childNodesPerBatch)
:
+                    t.getChildNodeNames(Integer.MAX_VALUE);
+            while (it.hasNext()) {
+                String name = it.next();
+                if (s.exists(name)) {
+                    // if it exists in the source, it's already updated
+                } else if (s.hasMoreChildNodes() && sourceMk.nodeExists(PathUtils.concat(source,
name), sourceRev)) {
+                    // if it exists in the source, it's already updated
+                    // (in this case, there are many child nodes)
+                } else {
+                    visit(PathUtils.concat(relPath, name));
+                }
             }
         }
     }
 
     /**
+     * Get a child node name iterator that batches node names. This work
+     * efficiently for small and big child node lists.
+     *
+     * @param mk the implementation
+     * @param path the path
+     * @param rev the revision
+     * @param batchSize the batch size
+     * @return a child node name iterator
+     */
+    public static Iterator<String> getAllChildNodeNames(final MicroKernel mk, final
String path, final String rev, final int batchSize) {
+        return new Iterator<String>() {
+
+            private long offset;
+            private Iterator<String> current;
+
+            {
+                nextBatch();
+            }
+
+            private void nextBatch() {
+                NodeImpl n = NodeImpl.parse(mk.getNodes(path, rev, 0, offset, batchSize));
+                current = n.getChildNodeNames(Integer.MAX_VALUE);
+                offset += batchSize;
+            }
+
+            @Override
+            public boolean hasNext() {
+                if (!current.hasNext()) {
+                    nextBatch();
+                }
+                return current.hasNext();
+            }
+
+            @Override
+            public String next() {
+                if (!current.hasNext()) {
+                    nextBatch();
+                }
+                return current.next();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    /**
      * The sync handler.
      */
     public interface Handler {

Modified: jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java?rev=1243562&r1=1243561&r2=1243562&view=diff
==============================================================================
--- jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java
(original)
+++ jackrabbit/sandbox/microkernel/src/test/java/org/apache/jackrabbit/mk/util/SyncTest.java
Mon Feb 13 15:32:39 2012
@@ -14,6 +14,9 @@
 package org.apache.jackrabbit.mk.util;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import java.util.HashSet;
+import java.util.Iterator;
 import org.apache.jackrabbit.mk.MultiMkTestBase;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -30,9 +33,37 @@ public class SyncTest extends MultiMkTes
     }
 
     @Test
+    public void testIterator() {
+        HashSet<String> names = new HashSet<String>();
+        for (int i = 0; i < 20; i++) {
+            mk.commit("/", "+ \"n" + i + "\": {}", mk.getHeadRevision(), "");
+            names.add("n" + i);
+        }
+        String head = mk.getHeadRevision();
+        Iterator<String> it = Sync.getAllChildNodeNames(mk, "/", head, 2);
+        while (it.hasNext()) {
+            String n = it.next();
+            assertTrue(names.remove(n));
+        }
+        assertEquals(0, names.size());
+    }
+
+    @Test
     public void test() {
+        doTest(-1);
+    }
+
+    @Test
+    public void testSmallChildNodeBatchSize() {
+        doTest(1);
+    }
+
+    private void doTest(int childNodeBatchSize) {
         mk.commit("/", "+ \"source\": { \"id\": 1, \"plus\": 0, \"a\": { \"x\": 10, \"y\":
20 }, \"b\": {\"z\": 100}, \"d\":{} }", mk.getHeadRevision(), "");
         Sync sync = new Sync();
+        if (childNodeBatchSize > 0) {
+            sync.setChildNodesPerBatch(childNodeBatchSize);
+        }
         String head = mk.getHeadRevision();
         sync.setSource(mk, head, "/");
         String diff = syncToString(sync);



Mime
View raw message