jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mdue...@apache.org
Subject svn commit: r1309895 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/kernel/ test/java/org/apache/jackrabbit/oak/kernel/
Date Thu, 05 Apr 2012 15:15:57 GMT
Author: mduerig
Date: Thu Apr  5 15:15:57 2012
New Revision: 1309895

URL: http://svn.apache.org/viewvc?rev=1309895&view=rev
Log:
OAK-9: Internal tree builder
Remove offset and size parameters from getChildNodes and replace with page wise loading from
backend

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/kernel/TransientNodeState.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorFuzzTest.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/kernel/TransientNodeState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/kernel/TransientNodeState.java?rev=1309895&r1=1309894&r2=1309895&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/kernel/TransientNodeState.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/kernel/TransientNodeState.java
Thu Apr  5 15:15:57 2012
@@ -25,15 +25,21 @@ import org.apache.commons.collections.it
 import org.apache.jackrabbit.mk.model.ChildNodeEntry;
 import org.apache.jackrabbit.mk.model.NodeState;
 import org.apache.jackrabbit.mk.model.PropertyState;
+import org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.PagedIterator;
 
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
-import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.*;
+import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.Function1;
+import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.Predicate;
+import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.add;
+import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.filter;
+import static org.apache.jackrabbit.oak.kernel.TransientNodeState.Iterators.map;
 
 /**
  * A transient node state represents a node being edited. All edit operations are
@@ -286,12 +292,7 @@ public class TransientNodeState {
      * the returned iterable.
      * @return  An {@code Iterable} for all child node states
      */
-    public Iterable<TransientNodeState> getChildNodes(long offset, int count) {
-        // Persisted child node states
-        final Iterable<? extends ChildNodeEntry> persisted = persistentState == null
-            ? null
-            : persistentState.getChildNodeEntries(offset, count);
-
+    public Iterable<TransientNodeState> getChildNodes() {
         // Copy od removed child node states
         final Set<String> removed = new HashSet<String>();
         removed.addAll(removedNodes);
@@ -307,13 +308,12 @@ public class TransientNodeState {
             @Override
             public Iterator<TransientNodeState> iterator() {
                 // persisted entries
-                Iterator<? extends ChildNodeEntry> nodes = persisted == null
-                    ? Iterators.<ChildNodeEntry>empty()
-                    : persisted.iterator();
+                final Iterator<? extends ChildNodeEntry> persisted =
+                    getPersistedChildNodeEntries(persistentState);
 
                 // persisted entries - removed entries
                 Iterator<ChildNodeEntry> persistedMinusRemovedEntries =
-                    filter(nodes, new Predicate<ChildNodeEntry>() {
+                    filter(persisted, new Predicate<ChildNodeEntry>() {
                         @Override
                         public boolean evaluate(ChildNodeEntry entry) {
                             return !removed.contains(entry.getName());
@@ -498,6 +498,30 @@ public class TransientNodeState {
         return persistentState != null && persistentState.getProperty(name) != null;
     }
 
+    /**
+     * Iterator over all persisted child node entries of the given
+     * {@code persistentState}. This iterator reads the child node entries page wise
+     * with a page size of 1024 items.
+     * @param persistentState  persistent state for retrieving the child node entries from
+     * @return  iterator of child node entries
+     */
+    private static Iterator<? extends ChildNodeEntry> getPersistedChildNodeEntries(
+            final NodeState persistentState) {
+
+        if (persistentState == null) {
+            return Iterators.empty();
+        }
+        else {
+            return Iterators.flatten(
+                new PagedIterator<ChildNodeEntry>(1024) {
+                    @Override
+                    protected Iterator<? extends ChildNodeEntry> getPage(long pos,
int size) {
+                        return persistentState.getChildNodeEntries(pos, size).iterator();
+                    }
+                });
+        }
+    }
+
     // TODO: move to a more suitable location
     static final class Iterators {
         private Iterators() { }
@@ -591,5 +615,108 @@ public class TransientNodeState {
         public interface Function1<S, T> {
             T apply(S argument);
         }
+
+        /**
+         * Flattens an iterator of iterators into a single iterator.
+         * @param iterators
+         * @param <T>
+         * @return
+         */
+        public static <T> Iterator<? extends T> flatten(
+                final Iterator<Iterator<? extends T>> iterators) {
+
+            return new Iterator<T>() {
+                private Iterator<? extends T> current;
+
+                @Override
+                public boolean hasNext() {
+                    if (current != null && current.hasNext()) {
+                        return true;
+                    }
+                    else if (!iterators.hasNext()) {
+                        return false;
+                    }
+                    else {
+                        do {
+                            current = iterators.next();
+                        } while (!current.hasNext() && iterators.hasNext());
+                        return current.hasNext();
+                    }
+                }
+
+                @Override
+                public T next() {
+                    if (!hasNext()) {
+                        throw new NoSuchElementException();
+                    }
+
+                    return current.next();
+                }
+
+                @Override
+                public void remove() {
+                    if (current == null) {
+                        throw new IllegalStateException();
+                    }
+
+                    current.remove();
+                }
+            };
+        }
+
+        /**
+         * A {@code PagedIterator} is an iterator of several pages. A page itself is
+         * an iterator. The abstract {@code getPage} method is called whenever this
+         * iterator needs to fetch another page.<p/>
+         *
+         * Lazy flattening (e.g. with {@link Iterators#flatten(java.util.Iterator)}
+         * results in an iterator which does batch reading from its back end.
+         *
+         * @param <T>
+         */
+        public abstract static class PagedIterator<T>
+                implements Iterator<Iterator<? extends T>> {
+
+            private final int pageSize;
+            private long pos;
+            private Iterator<? extends T> current;
+
+            protected PagedIterator(int pageSize) {
+                this.pageSize = pageSize;
+            }
+
+            /**
+             * @param pos  start index
+             * @param size  maximal number of elements
+             * @return  iterator starting at index {@code pos} containing at most {@code
size} elements.
+             */
+            protected abstract Iterator<? extends T> getPage(long pos, int size);
+
+            @Override
+            public boolean hasNext() {
+                if (current == null) {
+                    current = getPage(pos, pageSize);
+                    pos += pageSize;
+                }
+
+                return current.hasNext();
+            }
+
+            @Override
+            public Iterator<? extends T> next() {
+                if (!hasNext()) {
+                    throw new NoSuchElementException();
+                }
+                Iterator<? extends T> e = current;
+                current = null;
+                return e;
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException("remove");
+            }
+        }
+
     }
 }

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorFuzzTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorFuzzTest.java?rev=1309895&r1=1309894&r2=1309895&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorFuzzTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorFuzzTest.java
Thu Apr  5 15:15:57 2012
@@ -439,7 +439,7 @@ public class KernelNodeStateEditorFuzzTe
             assertEquals(property1, state2.getProperty(property1.getName()));
         }
 
-        for (TransientNodeState node1 : state1.getChildNodes(0, -1)) {
+        for (TransientNodeState node1 : state1.getChildNodes()) {
             checkEqual(node1, state2.getChildNode(node1.getName()));
         }
     }

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorTest.java?rev=1309895&r1=1309894&r2=1309895&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/kernel/KernelNodeStateEditorTest.java
Thu Apr  5 15:15:57 2012
@@ -90,7 +90,7 @@ public class KernelNodeStateEditorTest {
     public void getNodes() {
         KernelNodeStateEditor editor = new KernelNodeStateEditor(state);
         TransientNodeState transientState = editor.getTransientState();
-        Iterable<TransientNodeState> nodes = transientState.getChildNodes(0, -1);
+        Iterable<TransientNodeState> nodes = transientState.getChildNodes();
 
         Set<String> expectedPaths = new HashSet<String>();
         Collections.addAll(expectedPaths, "x", "y", "z");
@@ -291,5 +291,26 @@ public class KernelNodeStateEditorTest {
         transientState.setProperty(new KernelPropertyState("a", value));
         assertEquals(4, transientState.getPropertyCount());
     }
+    
+    @Test
+    public void largeChildNodeList() {
+        KernelNodeStateEditor editor = new KernelNodeStateEditor(state);
+
+        editor.addNode("large");
+        editor = editor.edit("large");
+        for (int c = 0; c < 10000; c++) {
+            editor.addNode("n" + c);
+        }
+
+        KernelNodeState newState = editor.mergeInto(microkernel, state);
+        editor = new KernelNodeStateEditor(newState);
+        editor = editor.edit("large");
+
+        int c = 0;
+        for (TransientNodeState q : editor.getTransientState().getChildNodes()) {
+            assertEquals("n" + c++, q.getName());
+        }
+
+    }
 
 }



Mime
View raw message