jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mreut...@apache.org
Subject svn commit: r1523568 - in /jackrabbit/oak/trunk/oak-core/src: main/java/org/apache/jackrabbit/oak/plugins/mongomk/ test/java/org/apache/jackrabbit/oak/plugins/mongomk/
Date Mon, 16 Sep 2013 09:16:20 GMT
Author: mreutegg
Date: Mon Sep 16 09:16:19 2013
New Revision: 1523568

URL: http://svn.apache.org/r1523568
Log:
OAK-926: MongoMK: split documents when they are too large
- Split off DELETED and COMMIT_ROOT as well

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
    jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
Mon Sep 16 09:16:19 2013
@@ -244,8 +244,11 @@ public class Commit {
                 }
             }
             for (UpdateOp op : changedNodes) {
-                // set commit root on changed nodes
-                NodeDocument.setCommitRoot(op, revision, commitRootDepth);
+                // set commit root on changed nodes unless it's the
+                // commit root itself
+                if (op != commitRoot) {
+                    NodeDocument.setCommitRoot(op, revision, commitRootDepth);
+                }
                 opLog.add(op);
                 createOrUpdateNode(store, op);
             }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
Mon Sep 16 09:16:19 2013
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -150,19 +151,19 @@ public class NodeDocument extends Docume
 
     /**
      * Returns <code>true</code> if the given <code>revision</code>
is marked
-     * committed in <strong>this</strong> document.
+     * committed in <strong>this</strong> document including previous documents.
      *
      * @param revision the revision.
      * @return <code>true</code> if committed; <code>false</code>
otherwise.
      */
     public boolean isCommitted(@Nonnull Revision revision) {
         String rev = checkNotNull(revision).toString();
-        String value = getRevisions().get(rev);
+        String value = getLocalRevisions().get(rev);
         if (value != null) {
             return Utils.isCommitted(value);
         }
         // check previous docs
-        for (NodeDocument prev : getPreviousDocs(revision)) {
+        for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
             if (prev.containsRevision(revision)) {
                 return prev.isCommitted(revision);
             }
@@ -185,7 +186,7 @@ public class NodeDocument extends Docume
         if (getLocalRevisions().containsKey(rev)) {
             return true;
         }
-        for (NodeDocument prev : getPreviousDocs(revision)) {
+        for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
             if (prev.containsRevision(revision)) {
                 return true;
             }
@@ -240,11 +241,12 @@ public class NodeDocument extends Docume
      */
     @CheckForNull
     public String getCommitRootPath(String revision) {
-        Map<String, Integer> valueMap = getCommitRoot();
-        Integer depth = valueMap.get(revision);
+        Map<String, String> valueMap = getCommitRoot();
+        String depth = valueMap.get(revision);
         if (depth != null) {
             String p = Utils.getPathFromId(getId());
-            return PathUtils.getAncestorPath(p, PathUtils.getDepth(p) - depth);
+            return PathUtils.getAncestorPath(p,
+                    PathUtils.getDepth(p) - Integer.parseInt(depth));
         } else {
             return null;
         }
@@ -581,6 +583,10 @@ public class NodeDocument extends Docume
      */
     @Nonnull
     public Iterable<UpdateOp> split(@Nonnull RevisionContext context) {
+        // only consider if there are enough commits
+        if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE)
{
+            return Collections.emptyList();
+        }
         String id = getId();
         SortedMap<Revision, Range> previous = getPreviousRanges();
         // what's the most recent previous revision?
@@ -594,47 +600,68 @@ public class NodeDocument extends Docume
                 recentPrevious = rev;
             }
         }
-        NavigableMap<Revision, String> splitRevs
-                = new TreeMap<Revision, String>(context.getRevisionComparator());
-        Map<String, String> revisions = getLocalRevisions();
-        // only consider if there are enough revisions
-        if (revisions.size() > REVISIONS_SPLIT_OFF_SIZE) {
-            // collect commits of this cluster node after the
+        Map<String, NavigableMap<Revision, String>> splitValues
+                = new HashMap<String, NavigableMap<Revision, String>>();
+        for (String property : new String[]{REVISIONS, COMMIT_ROOT, DELETED}) {
+            NavigableMap<Revision, String> splitMap
+                    = new TreeMap<Revision, String>(context.getRevisionComparator());
+            splitValues.put(property, splitMap);
+            Map<String, String> valueMap = getLocalMap(property);
+            // collect committed changes of this cluster node after the
             // most recent previous split revision
-            for (Map.Entry<String, String> entry : revisions.entrySet()) {
+            for (Map.Entry<String, String> entry : valueMap.entrySet()) {
                 Revision rev = Revision.fromString(entry.getKey());
                 if (rev.getClusterId() != context.getClusterId()) {
                     continue;
                 }
                 if (recentPrevious == null
                         || isRevisionNewer(context, rev, recentPrevious)) {
-                    String commitValue = entry.getValue();
-                    if (Utils.isCommitted(commitValue)) {
-                        splitRevs.put(rev, commitValue);
+                    if (isCommitted(rev)) {
+                        splitMap.put(rev, entry.getValue());
                     }
                 }
             }
         }
+
         List<UpdateOp> splitOps = Collections.emptyList();
-        if (splitRevs.size() > REVISIONS_SPLIT_OFF_SIZE) {
+        int numValues = 0;
+        Revision high = null;
+        Revision low = null;
+        for (NavigableMap<Revision, String> splitMap : splitValues.values()) {
+            // keep the most recent in the main document
+            if (!splitMap.isEmpty()) {
+                splitMap.remove(splitMap.lastKey());
+            }
+            if (splitMap.isEmpty()) {
+                continue;
+            }
+            // remember highest / lowest revision
+            if (high == null || isRevisionNewer(context, splitMap.lastKey(), high)) {
+                high = splitMap.lastKey();
+            }
+            if (low == null || isRevisionNewer(context, low, splitMap.firstKey())) {
+                low = splitMap.firstKey();
+            }
+            numValues += splitMap.size();
+        }
+        if (high != null && low != null && numValues >= REVISIONS_SPLIT_OFF_SIZE)
{
             // enough revisions to split off
             splitOps = new ArrayList<UpdateOp>(2);
-            // keep the most recent in the main document
-            splitRevs.remove(splitRevs.lastKey());
-            // move the others to another document
-            Revision high = splitRevs.lastEntry().getKey();
-            Revision low = splitRevs.firstEntry().getKey();
+            // move to another document
             UpdateOp main = new UpdateOp(id, false);
             main.setMapEntry(PREVIOUS, high.toString(), low.toString());
             UpdateOp old = new UpdateOp(Utils.getPreviousIdFor(id, high), true);
             old.set(ID, old.getKey());
-            for (Map.Entry<Revision, String> entry : splitRevs.entrySet()) {
-                String r = entry.getKey().toString();
-                main.removeMapEntry(REVISIONS, r);
-                old.setMapEntry(REVISIONS, r, entry.getValue());
+            for (String property : splitValues.keySet()) {
+                NavigableMap<Revision, String> splitMap = splitValues.get(property);
+                for (Map.Entry<Revision, String> entry : splitMap.entrySet()) {
+                    String r = entry.getKey().toString();
+                    main.removeMapEntry(property, r);
+                    old.setMapEntry(property, r, entry.getValue());
+                }
+                splitOps.add(old);
+                splitOps.add(main);
             }
-            splitOps.add(old);
-            splitOps.add(main);
         }
         return splitOps;
     }
@@ -688,20 +715,25 @@ public class NodeDocument extends Docume
     }
 
     /**
-     * Returns previous {@link NodeDocument}, which include the given revision.
+     * Returns previous {@link NodeDocument}, which include entries for the
+     * property in the given revision.
      * If the <code>revision</code> is <code>null</code>, then all
previous
      * documents are returned.
      *
      * @param revision the revision to match or <code>null</code>.
+     * @param property the name of a property.
      * @return previous documents.
      */
-    Iterable<NodeDocument> getPreviousDocs(final @Nullable Revision revision) {
+    Iterable<NodeDocument> getPreviousDocs(final @Nullable Revision revision,
+                                           final @Nonnull String property) {
+        checkNotNull(property);
         Iterable<NodeDocument> docs = Iterables.transform(
                 Iterables.filter(getPreviousRanges().entrySet(),
                         new Predicate<Map.Entry<Revision, Range>>() {
                             @Override
                             public boolean apply(Map.Entry<Revision, Range> input)
{
-                                return revision == null || input.getValue().includes(revision);
+                                return revision == null
+                                        || input.getValue().includes(revision);
                             }
                         }), new Function<Map.Entry<Revision, Range>, NodeDocument>()
{
             @Nullable
@@ -723,7 +755,8 @@ public class NodeDocument extends Docume
                 if (input == null) {
                     return false;
                 }
-                return revision == null || input.containsRevision(revision.toString());
+                return revision == null
+                        || input.getLocalMap(property).containsKey(revision.toString());
             }
         });
     }
@@ -753,6 +786,16 @@ public class NodeDocument extends Docume
         return getLocalMap(REVISIONS);
     }
 
+    @Nonnull
+    Map<String, String> getLocalCommitRoot() {
+        return getLocalMap(COMMIT_ROOT);
+    }
+
+    @Nonnull
+    Map<String, String> getLocalDeleted() {
+        return getLocalMap(DELETED);
+    }
+
     //-------------------------< UpdateOp modifiers >---------------------------
 
     public static void setModified(@Nonnull UpdateOp op,
@@ -784,7 +827,7 @@ public class NodeDocument extends Docume
                                      @Nonnull Revision revision,
                                      int commitRootDepth) {
         checkNotNull(op).setMapEntry(COMMIT_ROOT,
-                checkNotNull(revision).toString(), commitRootDepth);
+                checkNotNull(revision).toString(), String.valueOf(commitRootDepth));
     }
 
     public static void setDeleted(@Nonnull UpdateOp op,
@@ -807,14 +850,7 @@ public class NodeDocument extends Docume
         if (containsRevision(rev)) {
             return this;
         }
-        // check commit root
-        Map<String, Integer> commitRoot = getCommitRoot();
-        String commitRootPath = null;
-        Integer depth = commitRoot.get(rev.toString());
-        if (depth != null) {
-            String p = Utils.getPathFromId(getId());
-            commitRootPath = PathUtils.getAncestorPath(p, PathUtils.getDepth(p) - depth);
-        }
+        String commitRootPath = getCommitRootPath(rev.toString());
         if (commitRootPath == null) {
             // shouldn't happen, either node is commit root for a revision
             // or has a reference to the commit root
@@ -898,7 +934,7 @@ public class NodeDocument extends Docume
         String value = getLocalRevisions().get(r);
         if (value == null) {
             // check previous
-            for (NodeDocument prev : getPreviousDocs(revision)) {
+            for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
                 value = prev.getLocalRevisions().get(r);
                 if (value != null) {
                     break;
@@ -946,9 +982,9 @@ public class NodeDocument extends Docume
      */
     @CheckForNull
     private static String getLatestValue(@Nonnull RevisionContext context,
-                                  @Nonnull Map<String, String> valueMap,
-                                  @Nullable Revision min,
-                                  @Nonnull Revision max) {
+                                         @Nonnull Map<String, String> valueMap,
+                                         @Nullable Revision min,
+                                         @Nonnull Revision max) {
         String value = null;
         Revision latestRev = null;
         for (String r : valueMap.keySet()) {
@@ -978,12 +1014,7 @@ public class NodeDocument extends Docume
     }
 
     @Nonnull
-    private Map<String, Integer> getCommitRoot() {
-        @SuppressWarnings("unchecked")
-        Map<String, Integer> commitRoot = (Map<String, Integer>) get(COMMIT_ROOT);
-        if (commitRoot == null) {
-            commitRoot = Collections.emptyMap();
-        }
-        return commitRoot;
+    private Map<String, String> getCommitRoot() {
+        return ValueMap.create(this, COMMIT_ROOT);
     }
 }

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
Mon Sep 16 09:16:19 2013
@@ -46,7 +46,7 @@ class ValueMap {
             @Nonnull
             public Iterator<Map.Entry<String, String>> iterator() {
                 return Iterators.concat(map.entrySet().iterator(), Iterators.concat(new Iterator<Iterator<Map.Entry<String,
String>>>() {
-                    private final Iterator<NodeDocument> previous = doc.getPreviousDocs(null).iterator();
+                    private final Iterator<NodeDocument> previous = doc.getPreviousDocs(null,
property).iterator();
 
                     @Override
                     public boolean hasNext() {
@@ -68,7 +68,7 @@ class ValueMap {
             @Override
             public int size() {
                 int size = map.size();
-                for (NodeDocument prev : doc.getPreviousDocs(null)) {
+                for (NodeDocument prev : doc.getPreviousDocs(null, property)) {
                     size += prev.getLocalMap(property).size();
                 }
                 return size;
@@ -92,7 +92,7 @@ class ValueMap {
                     return value;
                 }
                 Revision r = Revision.fromString(key.toString());
-                for (NodeDocument prev : doc.getPreviousDocs(r)) {
+                for (NodeDocument prev : doc.getPreviousDocs(r, property)) {
                     value = prev.getLocalMap(property).get(key);
                     if (value != null) {
                         return value;

Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java
Mon Sep 16 09:16:19 2013
@@ -26,6 +26,7 @@ import com.google.common.collect.Sets;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -40,16 +41,11 @@ public class DocumentSplitTest extends B
         NodeDocument doc = store.find(Collection.NODES, Utils.getIdFromPath("/"));
         assertNotNull(doc);
         revisions.addAll(doc.getLocalRevisions().keySet());
-        
-        // MongoMK initializes with a root node with a single revision
-        int numRevs = 1; 
-        
         revisions.add(mk.commit("/", "+\"foo\":{}+\"bar\":{}", null, null));
-        numRevs++;
         // create nodes
-        while (numRevs++ <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
-            revisions.add(mk.commit("/", "+\"foo/node-" + numRevs + "\":{}" +
-                    "+\"bar/node-" + numRevs + "\":{}", null, null));
+        while (revisions.size() <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
+            revisions.add(mk.commit("/", "+\"foo/node-" + revisions.size() + "\":{}" +
+                    "+\"bar/node-" + revisions.size() + "\":{}", null, null));
         }
         mk.runBackgroundOperations();
         String head = mk.getHeadRevision();
@@ -69,4 +65,42 @@ public class DocumentSplitTest extends B
         mk.setAsyncDelay(0);
         mk.backgroundWrite();
     }
+
+    @Test
+    public void splitDeleted() throws Exception {
+        DocumentStore store = mk.getDocumentStore();
+        Set<String> revisions = Sets.newHashSet();
+        mk.commit("/", "+\"foo\":{}", null, null);
+        NodeDocument doc = store.find(Collection.NODES, Utils.getIdFromPath("/foo"));
+        assertNotNull(doc);
+        revisions.addAll(doc.getLocalRevisions().keySet());
+        boolean create = false;
+        while (revisions.size() <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
+            if (create) {
+                revisions.add(mk.commit("/", "+\"foo\":{}", null, null));
+            } else {
+                revisions.add(mk.commit("/", "-\"foo\"", null, null));
+            }
+            create = !create;
+        }
+        mk.runBackgroundOperations();
+        String head = mk.getHeadRevision();
+        doc = store.find(Collection.NODES, Utils.getIdFromPath("/foo"));
+        assertNotNull(doc);
+        Map<String, String> deleted = doc.getLocalDeleted();
+        // one remaining in the local deleted map
+        assertEquals(1, deleted.size());
+        for (String r : revisions) {
+            Revision rev = Revision.fromString(r);
+            assertTrue(doc.containsRevision(rev));
+            assertTrue(doc.isCommitted(rev));
+        }
+        Node node = doc.getNodeAtRevision(mk, Revision.fromString(head));
+        // check status of node
+        if (create) {
+            assertNull(node);
+        } else {
+            assertNotNull(node);
+        }
+    }
 }



Mime
View raw message