jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chet...@apache.org
Subject svn commit: r1491442 - in /jackrabbit/oak/trunk/oak-mongomk/src: main/java/org/apache/jackrabbit/mongomk/ main/java/org/apache/jackrabbit/mongomk/osgi/ main/java/org/apache/jackrabbit/mongomk/util/ main/resources/OSGI-INF/metatype/ test/java/org/apache...
Date Mon, 10 Jun 2013 13:18:06 GMT
Author: chetanm
Date: Mon Jun 10 13:18:05 2013
New Revision: 1491442

URL: http://svn.apache.org/r1491442
Log:
OAK-854 - Use weight based caching limit instead of size based limits

Adding support for using a Weigher which determines object weight through its size. The size
determination logic uses emperical data to determine the size. Following changes done
 - Exposed a config for cache size. Defaults to 256 MB
 - The cache size is then distributed between three caches Node, Children and Document in
20%, 10%, 70% respectively
 - Enabled stats collection by default. Later would make it configurable

Added:
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
  (with props)
Modified:
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
    jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
    jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java

Added: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java?rev=1491442&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
(added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
Mon Jun 10 13:18:05 2013
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.mongomk;
+
+import com.google.common.cache.Weigher;
+import org.apache.jackrabbit.mongomk.util.Utils;
+
+/**
+ * Determines the weight of object based on the memory taken by them. The memory esimates
+ * are based on emperical data and not exact
+ */
+public class EmpericalWeigher implements Weigher<String,Object>{
+
+    @Override
+    public int weigh(String key, Object value) {
+        int size = key.length() * 2;
+
+        if(value instanceof Node){
+            size += ((Node) value).getMemory();
+        }else if (value instanceof Node.Children){
+            size += ((Node.Children) value).getMemory();
+        }else if(value instanceof MongoDocumentStore.CachedDocument){
+            size += Utils.estimateMemoryUsage(((MongoDocumentStore.CachedDocument) value).value);
+        }else if(value != null){
+            throw new IllegalArgumentException("Cannot determine weight for object of type
" + value.getClass());
+        }
+        return size;
+    }
+}

Propchange: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
Mon Jun 10 13:18:05 2013
@@ -52,12 +52,6 @@ public class MongoDocumentStore implemen
 
     private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class);
     
-    /**
-     * The number of documents to cache.
-     */
-    private static final int CACHE_DOCUMENTS = Integer.getInteger(
-            "oak.mongoMK.cacheDocs", 10 * 1024);
-
     private static final boolean LOG_TIME = false;
 
     private final DBCollection nodes;
@@ -70,7 +64,7 @@ public class MongoDocumentStore implemen
     
     private final Cache<String, CachedDocument> nodesCache;
 
-    public MongoDocumentStore(DB db) {
+    public MongoDocumentStore(DB db,MongoMK.Builder builder) {
         nodes = db.getCollection(
                 Collection.NODES.toString());
         clusterNodes = db.getCollection(
@@ -87,7 +81,9 @@ public class MongoDocumentStore implemen
 
         // TODO expire entries if the parent was changed
         nodesCache = CacheBuilder.newBuilder()
-                .maximumSize(CACHE_DOCUMENTS)
+                .weigher(builder.getWeigher())
+                .recordStats()
+                .maximumWeight(builder.getDocumentCacheSize())
                 .build();
         
     }

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
Mon Jun 10 13:18:05 2013
@@ -38,6 +38,7 @@ import javax.annotation.Nullable;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
 import org.apache.jackrabbit.mk.api.MicroKernel;
 import org.apache.jackrabbit.mk.api.MicroKernelException;
 import org.apache.jackrabbit.mk.blobs.BlobStore;
@@ -66,18 +67,6 @@ public class MongoMK implements MicroKer
     private static final Logger LOG = LoggerFactory.getLogger(MongoMK.class);
 
     /**
-     * The number of child node list entries to cache.
-     */
-    private static final int CACHE_CHILDREN = 
-            Integer.getInteger("oak.mongoMK.cacheChildren", 1024);
-    
-    /**
-     * The number of nodes to cache.
-     */
-    private static final int CACHE_NODES = 
-            Integer.getInteger("oak.mongoMK.cacheNodes", 5 * 1024);
-    
-    /**
      * The number of diffs to cache.
      */
     private static final int CACHE_DIFF = 
@@ -231,13 +220,19 @@ public class MongoMK implements MicroKer
         this.asyncDelay = builder.getAsyncDelay();
         this.branches = new UnmergedBranches(revisionComparator);
 
-        //TODO Use size based weigher
+        //TODO Make stats collection configurable as it add slight overhead
+        //TODO Expose the stats as JMX beans
+
         nodeCache = CacheBuilder.newBuilder()
-                        .maximumSize(CACHE_NODES)
+                        .weigher(builder.getWeigher())
+                        .maximumWeight(builder.getNodeCacheSize())
+                        .recordStats()
                         .build();
 
         nodeChildrenCache =  CacheBuilder.newBuilder()
-                        .maximumSize(CACHE_CHILDREN)
+                        .weigher(builder.getWeigher())
+                        .recordStats()
+                        .maximumWeight(builder.getChildrenCacheSize())
                         .build();
         
         diffCache = CacheBuilder.newBuilder()
@@ -1547,12 +1542,20 @@ public class MongoMK implements MicroKer
      * A builder for a MongoMK instance.
      */
     public static class Builder {
-        
+        private static final long DEFAULT_MEMORY_CACHE_SIZE = 1 << 28; // 256MB
         private DocumentStore documentStore;
         private BlobStore blobStore;
         private int clusterId  = Integer.getInteger("oak.mongoMK.clusterId", 0);
         private int asyncDelay = 1000;
         private boolean timing;
+        private Weigher<String,Object> weigher = new EmpericalWeigher();
+        private long nodeCacheSize;
+        private long childrenCacheSize;
+        private long documentCacheSize;
+
+        public Builder() {
+            memoryCacheSize(DEFAULT_MEMORY_CACHE_SIZE);
+        }
 
         /**
          * Set the MongoDB connection to use. By default an in-memory store is used.
@@ -1562,7 +1565,7 @@ public class MongoMK implements MicroKer
          */
         public Builder setMongoDB(DB db) {
             if (db != null) {
-                this.documentStore = new MongoDocumentStore(db);
+                this.documentStore = new MongoDocumentStore(db,this);
                 this.blobStore = new MongoBlobStore(db);
             }
             return this;
@@ -1650,7 +1653,35 @@ public class MongoMK implements MicroKer
         public int getAsyncDelay() {
             return asyncDelay;
         }
-        
+
+        public Weigher<String, Object> getWeigher() {
+            return weigher;
+        }
+
+        public Builder weigher(Weigher<String, Object> weigher) {
+            this.weigher = weigher;
+            return this;
+        }
+
+        public Builder memoryCacheSize(long memoryCacheSize) {
+            this.nodeCacheSize = memoryCacheSize * 20 / 100;
+            this.childrenCacheSize = memoryCacheSize * 10 / 100;
+            this.documentCacheSize = memoryCacheSize - nodeCacheSize - childrenCacheSize;
+            return this;
+        }
+
+        public long getNodeCacheSize() {
+            return nodeCacheSize;
+        }
+
+        public long getChildrenCacheSize() {
+            return childrenCacheSize;
+        }
+
+        public long getDocumentCacheSize() {
+            return documentCacheSize;
+        }
+
         /**
          * Open the MongoMK instance using the configured options.
          * 

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
Mon Jun 10 13:18:05 2013
@@ -126,6 +126,15 @@ public class Node {
             this.path = path;
             this.rev = rev;
         }
+
+        public int getMemory(){
+            int size = 200 + path.length() * 2;
+            for(String c : children){
+                size += c.length() * 2 + 40;
+            }
+            return size;
+        }
+
         
         @Override
         public String toString() {

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
Mon Jun 10 13:18:05 2013
@@ -51,6 +51,7 @@ public class MongoMicroKernelService {
     private static final String DEFAULT_HOST = "localhost";
     private static final int DEFAULT_PORT = 27017;
     private static final String DEFAULT_DB = "oak";
+    private static final int DEFAULT_CACHE = 256;
 
     @Property(value = DEFAULT_HOST)
     private static final String PROP_HOST = "host";
@@ -61,6 +62,10 @@ public class MongoMicroKernelService {
     @Property(value = DEFAULT_DB)
     private static final String PROP_DB = "db";
 
+    @Property(intValue=DEFAULT_CACHE)
+    private static final String PROP_CACHE = "cache";
+    private static final long MB = 1024 * 1024;
+
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     private ServiceRegistration reg;
@@ -72,6 +77,7 @@ public class MongoMicroKernelService {
         String host = PropertiesUtil.toString(config.get(PROP_HOST), DEFAULT_HOST);
         int port = PropertiesUtil.toInteger(config.get(PROP_PORT), DEFAULT_PORT);
         String db = PropertiesUtil.toString(config.get(PROP_DB), DEFAULT_DB);
+        int cacheSize = PropertiesUtil.toInteger(config.get(PROP_CACHE), DEFAULT_CACHE);
 
         logger.info("Starting MongoDB MicroKernel with host={}, port={}, db={}",
                 new Object[] {host, port, db});
@@ -81,7 +87,7 @@ public class MongoMicroKernelService {
 
         logger.info("Connected to database {}", mongoDB);
 
-        mk = new MongoMK.Builder().setMongoDB(mongoDB).open();
+        mk = new MongoMK.Builder().memoryCacheSize(cacheSize * MB).setMongoDB(mongoDB).open();
 
         Properties props = new Properties();
         props.setProperty("oak.mk.type", "mongo");

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
Mon Jun 10 13:18:05 2013
@@ -22,6 +22,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 
+import com.mongodb.BasicDBObject;
 import org.bson.types.ObjectId;
 
 /**
@@ -56,17 +57,26 @@ public class Utils {
             return 0;
         }
         int size = 0;
+
         for (Entry<String, Object> e : map.entrySet()) {
-            size += e.getKey().length();
+            size += e.getKey().length()*2;
             Object o = e.getValue();
             if (o instanceof String) {
-                size += o.toString().length();
+                size += ((String)o).length()*2;
             } else if (o instanceof Long) {
                 size += 8;
+            }  else if (o instanceof Integer) {
+                size += 4;
             } else if (o instanceof Map) {
                 size += 8 + estimateMemoryUsage((Map<String, Object>) o);
             }
         }
+
+        if(map instanceof BasicDBObject){
+            //Based on emperical testing using JAMM
+            size += 176;
+            size += map.entrySet().size() * 136;
+        }
         return size;
     }
 

Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
Mon Jun 10 13:18:05 2013
@@ -29,3 +29,6 @@ port.description = The port to connect t
 
 db.name = MongoDB Database
 db.description = The database to use.
+
+cache.name = Cache Size (MB)
+cache.description = Cache Size (MB)
\ No newline at end of file

Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
Mon Jun 10 13:18:05 2013
@@ -16,11 +16,14 @@
  */
 package org.apache.jackrabbit.mongomk;
 
+import static org.apache.jackrabbit.mongomk.Node.Children;
 import static org.junit.Assert.fail;
 
 import java.util.LinkedList;
 import java.util.concurrent.Callable;
 
+import com.mongodb.BasicDBObject;
+import org.apache.jackrabbit.mongomk.util.Utils;
 import org.junit.Test;
 
 /**
@@ -64,6 +67,50 @@ public class MeasureMemory {
             }
         });
     }
+
+    @Test
+    public void basicObject() throws Exception {
+        measureMemory(new Callable<Object[]>() {
+            @Override
+            public Object[] call() {
+                BasicDBObject n = generateBasicObject(15);
+                return new Object[]{n, Utils.estimateMemoryUsage(n) + OVERHEAD};
+            }
+        });
+    }
+
+    @Test
+    public void basicObjectWithoutProperties() throws Exception {
+        measureMemory(new Callable<Object[]>() {
+            @Override
+            public Object[] call() {
+                BasicDBObject n = generateBasicObject(0);
+                return new Object[]{n, Utils.estimateMemoryUsage(n) + OVERHEAD};
+            }
+        });
+    }
+
+    @Test
+    public void nodeChild() throws Exception {
+        measureMemory(new Callable<Object[]>() {
+            @Override
+            public Object[] call() {
+                Children n = generateNodeChild(15);
+                return new Object[]{n, n.getMemory() + OVERHEAD};
+            }
+        });
+    }
+
+    @Test
+    public void nodeChildWithoutChildren() throws Exception {
+        measureMemory(new Callable<Object[]>() {
+            @Override
+            public Object[] call() {
+                Children n = generateNodeChild(15);
+                return new Object[]{n, n.getMemory() + OVERHEAD};
+            }
+        });
+    }
     
     private static void measureMemory(Callable<Object[]> c) throws Exception {
         LinkedList<Object> list = new LinkedList<Object>();
@@ -99,6 +146,22 @@ public class MeasureMemory {
         n.setLastRevision(new Revision(1, 2, 3));
         return n;
     }
+
+    static BasicDBObject generateBasicObject(int propertyCount) {
+        BasicDBObject n = new BasicDBObject(new String("_id"),new String("/hello/world"));
+        for (int i = 0; i < propertyCount; i++) {
+            n.append("property" + i, "values " + i);
+        }
+        return n;
+    }
+
+    static Children generateNodeChild(int childCount) {
+        Children n = new Children(new String("_id"),new Revision(1, 2, 3));
+        for (int i = 0; i < childCount; i++) {
+            n.children.add("child"+i);
+        }
+        return n;
+    }
     
     private static long getMemoryUsed() {
         for (int i = 0; i < 10; i++) {

Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java
(original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java
Mon Jun 10 13:18:05 2013
@@ -54,7 +54,7 @@ public class MongoDocumentStoreTest {
 
     DocumentStore openDocumentStore() {
         if (MONGO_DB) {
-            return new MongoDocumentStore(MongoUtils.getConnection().getDB());
+            return new MongoDocumentStore(MongoUtils.getConnection().getDB(),new MongoMK.Builder());
         }
         return new MemoryDocumentStore();
     }



Mime
View raw message