jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mreut...@apache.org
Subject svn commit: r1526212 - in /jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk: MongoMK.java NodeDocument.java
Date Wed, 25 Sep 2013 15:58:43 GMT
Author: mreutegg
Date: Wed Sep 25 15:58:43 2013
New Revision: 1526212

URL: http://svn.apache.org/r1526212
Log:
OAK-1034: OOME running Oak benchmark suite
- split documents when they reach 1MB
- cache names of child nodes up to 16k

Modified:
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
    jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java?rev=1526212&r1=1526211&r2=1526212&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
Wed Sep 25 15:58:43 2013
@@ -93,7 +93,7 @@ public class MongoMK implements MicroKer
      * Do not cache more than this number of children for a document.
      */
     private static final int NUM_CHILDREN_CACHE_LIMIT = Integer.getInteger(
-            "oak.mongoMK.childrenCacheLimit", 1000);
+            "oak.mongoMK.childrenCacheLimit", 16 * 1024);
 
     /**
      * When trying to access revisions that are older than this many
@@ -617,7 +617,7 @@ public class MongoMK implements MicroKer
         }
         // check cache
         NodeDocument.Children c = docChildrenCache.getIfPresent(path);
-        if (c == null || (c.childNames.size() < limit && !c.isComplete)) {
+        if (c == null) {
             c = new NodeDocument.Children();
             List<NodeDocument> docs = store.query(Collection.NODES, from, to, limit);
             for (NodeDocument doc : docs) {
@@ -626,6 +626,22 @@ public class MongoMK implements MicroKer
             }
             c.isComplete = docs.size() < limit;
             docChildrenCache.put(path, c);
+        } else if (c.childNames.size() < limit && !c.isComplete) {
+            // fetch more and update cache
+            String lastName = c.childNames.get(c.childNames.size() - 1);
+            String lastPath = PathUtils.concat(path, lastName);
+            from = Utils.getIdFromPath(lastPath);
+            int remainingLimit = limit - c.childNames.size();
+            List<NodeDocument> docs = store.query(Collection.NODES,
+                    from, to, remainingLimit);
+            NodeDocument.Children clone = c.clone();
+            for (NodeDocument doc : docs) {
+                String p = Utils.getPathFromId(doc.getId());
+                clone.childNames.add(PathUtils.getName(p));
+            }
+            clone.isComplete = docs.size() < remainingLimit;
+            docChildrenCache.put(path, clone);
+            c = clone;
         }
         return Iterables.transform(c.childNames, new Function<String, NodeDocument>()
{
             @Override

Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java?rev=1526212&r1=1526211&r2=1526212&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
(original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
Wed Sep 25 15:58:43 2013
@@ -68,6 +68,12 @@ public class NodeDocument extends Docume
     static final int SPLIT_CANDIDATE_THRESHOLD = 32 * 1024;
 
     /**
+     * A document size threshold after which a split is forced even if
+     * {@link #REVISIONS_SPLIT_OFF_SIZE} is not reached.
+     */
+    static final int FORCE_SPLIT_THRESHOLD = 1024 * 1024;
+
+    /**
      * Only split off at least this number of revisions.
      */
     static final int REVISIONS_SPLIT_OFF_SIZE = 1000;
@@ -535,8 +541,10 @@ public class NodeDocument extends Docume
      */
     @Nonnull
     public Iterable<UpdateOp> split(@Nonnull RevisionContext context) {
-        // only consider if there are enough commits
-        if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE)
{
+        // only consider if there are enough commits,
+        // unless document is really big
+        if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE
+                && getMemory() < FORCE_SPLIT_THRESHOLD) {
             return Collections.emptyList();
         }
         String id = getId();
@@ -599,7 +607,8 @@ public class NodeDocument extends Docume
             }
             numValues += splitMap.size();
         }
-        if (high != null && low != null && numValues >= REVISIONS_SPLIT_OFF_SIZE)
{
+        if (high != null && low != null
+                && (numValues >= REVISIONS_SPLIT_OFF_SIZE || getMemory() >
FORCE_SPLIT_THRESHOLD)) {
             // enough revisions to split off
             splitOps = new ArrayList<UpdateOp>(2);
             // move to another document
@@ -963,12 +972,12 @@ public class NodeDocument extends Docume
      * The list of children for a node. The list might be complete or not, in
      * which case it only represents a block of children.
      */
-    static final class Children implements CacheValue {
+    static final class Children implements CacheValue, Cloneable {
 
         /**
          * The child node names, ordered as stored in MongoDB.
          */
-        final List<String> childNames = new ArrayList<String>();
+        ArrayList<String> childNames = new ArrayList<String>();
         
         /**
          * Whether the list is complete (in which case there are no other
@@ -984,6 +993,18 @@ public class NodeDocument extends Docume
             }
             return size;
         }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public Children clone() {
+            try {
+                Children clone = (Children) super.clone();
+                clone.childNames = (ArrayList<String>) childNames.clone();
+                return clone;
+            } catch (CloneNotSupportedException e) {
+                throw new RuntimeException();
+            }
+        }
     }
 
     /**



Mime
View raw message