jackrabbit-oak-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From thom...@apache.org
Subject svn commit: r1421124 - in /jackrabbit/oak/trunk/oak-mk: ./ src/main/java/org/apache/jackrabbit/mk/htree/ src/main/java/org/apache/jackrabbit/mk/model/ src/main/java/org/apache/jackrabbit/mk/store/
Date Thu, 13 Dec 2012 08:26:45 GMT
Author: thomasm
Date: Thu Dec 13 08:26:43 2012
New Revision: 1421124

URL: http://svn.apache.org/viewvc?rev=1421124&view=rev
Log:
OAK-512 Caching in MicroKernelImpl

Added:
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/CacheObject.java
Modified:
    jackrabbit/oak/trunk/oak-mk/pom.xml
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/ChildNodeEntriesHTree.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/HashDirectory.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractCommit.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractNode.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntries.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesMap.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesTree.java
    jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java

Modified: jackrabbit/oak/trunk/oak-mk/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/pom.xml?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-mk/pom.xml Thu Dec 13 08:26:43 2012
@@ -98,6 +98,13 @@
       <version>1.6.4</version>
     </dependency>
 
+    <!-- Caching -->    
+    <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>${guava.version}</version>
+    </dependency>
+    
     <!-- default persistence backend -->
     <dependency>
       <groupId>com.h2database</groupId>

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/ChildNodeEntriesHTree.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/ChildNodeEntriesHTree.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/ChildNodeEntriesHTree.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/ChildNodeEntriesHTree.java
Thu Dec 13 08:26:43 2012
@@ -176,4 +176,10 @@ public class ChildNodeEntriesHTree imple
 
         // nothing to be done here
     }
+
+    @Override
+    public int getMemory() {
+        return top.getMemory();
+    }
+
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/HashDirectory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/HashDirectory.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/HashDirectory.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/htree/HashDirectory.java
Thu Dec 13 08:26:43 2012
@@ -651,4 +651,13 @@ class HashDirectory implements ChildNode
     public Iterator<ChildNodeEntry> getRemoved(ChildNodeEntries other) {
         throw new NoSuchMethodError();
     }
+    
+    @Override
+    public int getMemory() {
+        // assuming a fixed size of 1000 entries, each with 100 bytes, plus 100
+        // bytes overhead
+        int memory = 100 + 1000 * 100;
+        return memory;
+    }
+    
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractCommit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractCommit.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractCommit.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractCommit.java
Thu Dec 13 08:26:43 2012
@@ -17,11 +17,12 @@
 package org.apache.jackrabbit.mk.model;
 
 import org.apache.jackrabbit.mk.store.Binding;
+import org.apache.jackrabbit.mk.store.CacheObject;
 
 /**
  *
  */
-public abstract class AbstractCommit implements Commit {
+public abstract class AbstractCommit implements Commit, CacheObject {
 
     // id of root node associated with this commit
     protected Id rootNodeId;
@@ -99,4 +100,17 @@ public abstract class AbstractCommit imp
         sb.append("branchRootId: '").append(branchRootId == null ? "" : branchRootId.toString()).append("'");
         return sb.toString();
     }
+    
+    @Override
+    public int getMemory() {
+        int memory = 100;
+        if (msg != null) {
+            memory += 2 * msg.length();
+        }
+        if (changes != null) {
+            memory += 2 * changes.length();
+        }
+        return memory;
+    }
+    
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractNode.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractNode.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/AbstractNode.java
Thu Dec 13 08:26:43 2012
@@ -17,16 +17,18 @@
 package org.apache.jackrabbit.mk.model;
 
 import org.apache.jackrabbit.mk.store.Binding;
+import org.apache.jackrabbit.mk.store.CacheObject;
 import org.apache.jackrabbit.mk.store.RevisionProvider;
 
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 
 /**
  *
  */
-public abstract class AbstractNode implements Node {
+public abstract class AbstractNode implements Node, CacheObject {
 
     protected RevisionProvider provider;
     
@@ -166,4 +168,16 @@ public abstract class AbstractNode imple
         binding.write(":inlined", childEntries.inlined() ? 1 : 0);
         childEntries.serialize(binding);
     }
+
+    @Override
+    public int getMemory() {
+        int memory = 100;
+        for (Entry<String, String> e : properties.entrySet()) {
+            memory += 2 * e.getKey().length();
+            memory += 2 * e.getValue().length();
+        }
+        
+        return memory;
+    }
+
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntries.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntries.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntries.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntries.java
Thu Dec 13 08:26:43 2012
@@ -17,13 +17,14 @@
 package org.apache.jackrabbit.mk.model;
 
 import org.apache.jackrabbit.mk.store.Binding;
+import org.apache.jackrabbit.mk.store.CacheObject;
 
 import java.util.Iterator;
 
 /**
  *
  */
-public interface ChildNodeEntries extends Cloneable {
+public interface ChildNodeEntries extends Cloneable, CacheObject {
     
     static final int CAPACITY_THRESHOLD = 1000;
 

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesMap.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesMap.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesMap.java
Thu Dec 13 08:26:43 2012
@@ -17,6 +17,7 @@
 package org.apache.jackrabbit.mk.model;
 
 import org.apache.jackrabbit.mk.store.Binding;
+import org.apache.jackrabbit.mk.store.CacheObject;
 import org.apache.jackrabbit.mk.util.AbstractFilteringIterator;
 import org.apache.jackrabbit.mk.util.RangeIterator;
 
@@ -29,7 +30,7 @@ import java.util.Map;
 /**
  *
  */
-public class ChildNodeEntriesMap implements ChildNodeEntries {
+public class ChildNodeEntriesMap implements ChildNodeEntries, CacheObject {
 
     protected static final List<ChildNodeEntry> EMPTY = Collections.emptyList();
     
@@ -219,4 +220,14 @@ public class ChildNodeEntriesMap impleme
         }
         return newInstance;
     }
+
+    @Override
+    public int getMemory() {
+        int memory = 100;
+        for (String e : entries.keySet()) {
+            memory += e.length() * 2 + 100;
+        }
+        return memory;
+    }
+    
 }

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesTree.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesTree.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesTree.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/model/ChildNodeEntriesTree.java
Thu Dec 13 08:26:43 2012
@@ -17,6 +17,7 @@
 package org.apache.jackrabbit.mk.model;
 
 import org.apache.jackrabbit.mk.store.Binding;
+import org.apache.jackrabbit.mk.store.CacheObject;
 import org.apache.jackrabbit.mk.store.RevisionProvider;
 import org.apache.jackrabbit.mk.store.RevisionStore;
 import org.apache.jackrabbit.mk.util.AbstractFilteringIterator;
@@ -32,7 +33,7 @@ import java.util.NoSuchElementException;
 /**
  *
  */
-public class ChildNodeEntriesTree implements ChildNodeEntries {
+public class ChildNodeEntriesTree implements ChildNodeEntries, CacheObject {
 
     protected static final List<ChildNodeEntry> EMPTY = Collections.emptyList();
     
@@ -688,4 +689,12 @@ public class ChildNodeEntriesTree implem
             return false;
         }
     }
+    
+    @Override
+    public int getMemory() {
+        // assuming a fixed size of 1000 entries, each with 100 bytes, plus 100 bytes overhead
+        int memory = 100 + 1000 * 100;
+        return memory;
+    }
+
 }

Added: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/CacheObject.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/CacheObject.java?rev=1421124&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/CacheObject.java
(added)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/CacheObject.java
Thu Dec 13 08:26:43 2012
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mk.store;
+
+/**
+ * An entry that can be put in a cache.
+ */
+public interface CacheObject {
+    
+    /**
+     * The estimated amount of memory used, in bytes.
+     * 
+     * @return the memory used
+     */
+    int getMemory();
+
+}

Modified: jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java?rev=1421124&r1=1421123&r2=1421124&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java
(original)
+++ jackrabbit/oak/trunk/oak-mk/src/main/java/org/apache/jackrabbit/mk/store/DefaultRevisionStore.java
Thu Dec 13 08:26:43 2012
@@ -31,10 +31,13 @@ import org.apache.jackrabbit.mk.model.tr
 import org.apache.jackrabbit.mk.persistence.GCPersistence;
 import org.apache.jackrabbit.mk.persistence.Persistence;
 import org.apache.jackrabbit.mk.util.IOUtils;
-import org.apache.jackrabbit.mk.util.SimpleLRUCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
+
 import java.io.Closeable;
 import java.util.Collections;
 import java.util.Iterator;
@@ -42,6 +45,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
 import java.util.WeakHashMap;
+import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
@@ -60,7 +64,9 @@ public class DefaultRevisionStore extend
     private static final Logger LOG = LoggerFactory.getLogger(DefaultRevisionStore.class);
     
     public static final String CACHE_SIZE = "mk.cacheSize";
-    public static final int DEFAULT_CACHE_SIZE = 10000;
+    
+    // default cache size is 32 MB
+    public static final int DEFAULT_CACHE_SIZE = 32 * 1024 * 1024;
 
     private boolean initialized;
     private Id head;
@@ -72,7 +78,7 @@ public class DefaultRevisionStore extend
     protected final GCPersistence gcpm;
 
     /* avoid synthetic accessor */ int initialCacheSize;
-    /* avoid synthetic accessor */ Map<Id, Object> cache;
+    /* avoid synthetic accessor */ Cache<Id, CacheObject> cache;
 
     /**
      * GC run state constants.
@@ -131,7 +137,15 @@ public class DefaultRevisionStore extend
         }
 
         initialCacheSize = determineInitialCacheSize();
-        cache = Collections.synchronizedMap(SimpleLRUCache.<Id, Object> newInstance(initialCacheSize));
+        
+        cache = CacheBuilder.newBuilder()
+                .maximumWeight(initialCacheSize)
+                .weigher(new Weigher<Id, CacheObject>() {
+                    public int weigh(Id id, CacheObject obj) {
+                        return obj.getMemory();
+                    }
+                })
+                .build();
 
         // make sure we've got a HEAD commit
         Id[] ids = pm.readIds();
@@ -184,7 +198,7 @@ public class DefaultRevisionStore extend
             gcExecutor.shutdown();
         }
 
-        cache.clear();
+        cache.invalidateAll();
 
         IOUtils.closeQuietly(pm);
 
@@ -354,49 +368,47 @@ public class DefaultRevisionStore extend
 
     // -----------------------------------------------------< RevisionProvider >
 
-    public StoredNode getNode(Id id) throws NotFoundException, Exception {
+    public StoredNode getNode(final Id id) throws NotFoundException, Exception {
         verifyInitialized();
 
-        StoredNode node = (StoredNode) cache.get(id);
-        if (node != null) {
-            return node;
-        }
-
-        node = new StoredNode(id, this);
-        pm.readNode(node);
-
-        cache.put(id, node);
-
+        StoredNode node = (StoredNode) cache.get(id,
+                new Callable<StoredNode>() {
+                    @Override
+                    public StoredNode call() throws Exception {
+                        StoredNode node = new StoredNode(id,
+                                DefaultRevisionStore.this);
+                        pm.readNode(node);
+                        return node;
+                    }
+                });
         return node;
     }
 
-    public ChildNodeEntriesMap getCNEMap(Id id) throws NotFoundException,
+    public ChildNodeEntriesMap getCNEMap(final Id id) throws NotFoundException,
             Exception {
         verifyInitialized();
 
-        ChildNodeEntriesMap map = (ChildNodeEntriesMap) cache.get(id);
-        if (map != null) {
-            return map;
-        }
-
-        map = pm.readCNEMap(id);
-
-        cache.put(id, map);
-
+        ChildNodeEntriesMap map = (ChildNodeEntriesMap) cache.get(id,
+                new Callable<ChildNodeEntriesMap>() {
+                    @Override
+                    public ChildNodeEntriesMap call() throws Exception {
+                        return pm.readCNEMap(id);
+                    }
+                });
         return map;
     }
 
-    public StoredCommit getCommit(Id id) throws NotFoundException, Exception {
+    public StoredCommit getCommit(final Id id) throws NotFoundException,
+            Exception {
         verifyInitialized();
 
-        StoredCommit commit = (StoredCommit) cache.get(id);
-        if (commit != null) {
-            return commit;
-        }
-
-        commit = pm.readCommit(id);
-        cache.put(id, commit);
-
+        StoredCommit commit = (StoredCommit) cache.get(id,
+                new Callable<StoredCommit>() {
+                    @Override
+                    public StoredCommit call() throws Exception {
+                        return pm.readCommit(id);
+                    }
+                });
         return commit;
     }
 
@@ -554,7 +566,7 @@ public class DefaultRevisionStore extend
         try {
             int swept = gcpm.sweep();
             LOG.debug("GC cycle swept {} items", swept);
-            cache.clear();
+            cache.invalidateAll();
         } catch (Exception e) {
             LOG.error("Exception occurred in GC cycle", e);
         } finally {



Mime
View raw message