jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mreut...@apache.org
Subject svn commit: r924677 - in /jackrabbit/trunk: jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integratio...
Date Thu, 18 Mar 2010 09:48:04 GMT
Author: mreutegg
Date: Thu Mar 18 09:48:03 2010
New Revision: 924677

URL: http://svn.apache.org/viewvc?rev=924677&view=rev
Log:
JCR-2524: Reduce memory usage of DocIds

Added:
    jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java
  (with props)
Modified:
    jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
    jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DocId.java
    jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/daily/DailyIntegrationTest.java
    jackrabbit/trunk/jackrabbit-spi-commons/src/main/java/org/apache/jackrabbit/spi/commons/name/HashCache.java

Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java?rev=924677&r1=924676&r2=924677&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
(original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
Thu Mar 18 09:48:03 2010
@@ -31,11 +31,13 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Map;
 import java.util.HashMap;
 import java.util.Collections;
 import java.text.NumberFormat;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Implements an <code>IndexReader</code> that maintains caches to resolve
@@ -61,14 +63,20 @@ class CachingIndexReader extends FilterI
     private final BitSet shareableNodes;
 
     /**
-     * Cache of nodes parent relation. If an entry in the array is not null,
-     * that means the node with the document number = array-index has the node
-     * with <code>DocId</code> as parent.
+     * Cache of nodes parent relation. If an entry in the array is >= 0,
+     * then that means the node with the document number = array-index has the
+     * node with the value at that position as parent.
      */
-    private final DocId[] parents;
+    private final int[] inSegmentParents;
 
     /**
-     * Initializes the {@link #parents} cache.
+     * Cache of nodes parent relation that point to a foreign index segment.
+     */
+    private final Map<Integer, DocId> foreignParentDocIds = new ConcurrentHashMap<Integer,
DocId>();
+
+    /**
+     * Initializes the {@link #inSegmentParents} and {@link #foreignParentDocIds}
+     * caches.
      */
     private CacheInitializer cacheInitializer;
 
@@ -99,7 +107,7 @@ class CachingIndexReader extends FilterI
      * @param delegatee the base <code>IndexReader</code>.
      * @param cache     a document number cache, or <code>null</code> if not
      *                  available to this reader.
-     * @param initCache if the {@link #parents} cache should be initialized
+     * @param initCache if the parent caches should be initialized
      *                  when this index reader is constructed.
      * @throws IOException if an error occurs while reading from the index.
      */
@@ -110,7 +118,8 @@ class CachingIndexReader extends FilterI
             throws IOException {
         super(delegatee);
         this.cache = cache;
-        this.parents = new DocId[delegatee.maxDoc()];
+        this.inSegmentParents = new int[delegatee.maxDoc()];
+        Arrays.fill(this.inSegmentParents, -1);
         this.shareableNodes = new BitSet();
         TermDocs tDocs = delegatee.termDocs(
                 new Term(FieldNames.SHAREABLE_NODE, ""));
@@ -144,7 +153,12 @@ class CachingIndexReader extends FilterI
     DocId getParent(int n, BitSet deleted) throws IOException {
         DocId parent;
         boolean existing = false;
-        parent = parents[n];
+        int parentDocNum = inSegmentParents[n];
+        if (parentDocNum != -1) {
+            parent = DocId.create(parentDocNum);
+        } else {
+            parent = foreignParentDocIds.get(n);
+        }
 
         if (parent != null) {
             existing = true;
@@ -159,6 +173,7 @@ class CachingIndexReader extends FilterI
         }
 
         if (parent == null) {
+            int plainDocId = -1;
             Document doc = document(n, FieldSelectors.UUID_AND_PARENT);
             String[] parentUUIDs = doc.getValues(FieldNames.PARENT);
             if (parentUUIDs.length == 0 || parentUUIDs[0].length() == 0) {
@@ -174,7 +189,8 @@ class CachingIndexReader extends FilterI
                         try {
                             while (docs.next()) {
                                 if (!deleted.get(docs.doc())) {
-                                    parent = DocId.create(docs.doc());
+                                    plainDocId = docs.doc();
+                                    parent = DocId.create(plainDocId);
                                     break;
                                 }
                             }
@@ -191,7 +207,20 @@ class CachingIndexReader extends FilterI
             }
 
             // finally put to cache
-            parents[n] = parent;
+            if (plainDocId != -1) {
+                // PlainDocId
+                inSegmentParents[n] = plainDocId;
+            } else {
+                // UUIDDocId
+                foreignParentDocIds.put(n, parent);
+                if (existing) {
+                    // there was an existing parent reference in
+                    // inSegmentParents, which was invalid and is replaced
+                    // with a UUIDDocId (points to a foreign segment).
+                    // mark as unknown
+                    inSegmentParents[n] = -1;
+                }
+            }
         }
         return parent;
     }
@@ -313,7 +342,8 @@ class CachingIndexReader extends FilterI
     }
 
     /**
-     * Initializes the {@link CachingIndexReader#parents} cache.
+     * Initializes the {@link CachingIndexReader#inSegmentParents} and
+     * {@link CachingIndexReader#foreignParentDocIds} caches.
      */
     private class CacheInitializer implements Runnable {
 
@@ -382,8 +412,8 @@ class CachingIndexReader extends FilterI
         }
 
         /**
-         * Initializes the {@link CachingIndexReader#parents} <code>DocId</code>
-         * array.
+         * Initializes the {@link CachingIndexReader#inSegmentParents} and
+         * {@link CachingIndexReader#foreignParentDocIds} caches.
          *
          * @param reader the underlying index reader.
          * @throws IOException if an error occurs while reading from the index.
@@ -432,28 +462,28 @@ class CachingIndexReader extends FilterI
             for (NodeInfo info : docs.values()) {
                 NodeInfo parent = docs.get(info.parent);
                 if (parent != null) {
-                    parents[info.docId] = DocId.create(parent.docId);
+                    inSegmentParents[info.docId] = parent.docId;
                 } else if (info.parent != null) {
                     foreignParents++;
-                    parents[info.docId] = DocId.create(info.parent);
+                    foreignParentDocIds.put(info.docId, DocId.create(info.parent));
                 } else if (shareableNodes.get(info.docId)) {
                     Document doc = reader.document(info.docId, FieldSelectors.UUID_AND_PARENT);
-                    parents[info.docId] = DocId.create(doc.getValues(FieldNames.PARENT));
+                    foreignParentDocIds.put(info.docId, DocId.create(doc.getValues(FieldNames.PARENT)));
                 } else {
                     // no parent -> root node
-                    parents[info.docId] = DocId.NULL;
+                    foreignParentDocIds.put(info.docId, DocId.NULL);
                 }
             }
             if (log.isDebugEnabled()) {
                 NumberFormat nf = NumberFormat.getPercentInstance();
                 nf.setMaximumFractionDigits(1);
                 time = System.currentTimeMillis() - time;
-                if (parents.length > 0) {
-                    foreignParents /= parents.length;
+                if (inSegmentParents.length > 0) {
+                    foreignParents /= inSegmentParents.length;
                 }
                 log.debug("initialized {} DocIds in {} ms, {} foreign parents",
                         new Object[]{
-                            parents.length,
+                            inSegmentParents.length,
                             time,
                             nf.format(foreignParents)
                         });

Modified: jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DocId.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DocId.java?rev=924677&r1=924676&r2=924677&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DocId.java
(original)
+++ jackrabbit/trunk/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/DocId.java
Thu Mar 18 09:48:03 2010
@@ -32,6 +32,17 @@ abstract class DocId {
     static final int[] EMPTY = new int[0];
 
     /**
+     * All DocIds with a value smaller than {@link Short#MAX_VALUE}.
+     */
+    private static final PlainDocId[] LOW_DOC_IDS = new PlainDocId[Short.MAX_VALUE];
+
+    static {
+        for (int i = 0; i < LOW_DOC_IDS.length; i++) {
+            LOW_DOC_IDS[i] = new PlainDocId(i);
+        }
+    }
+
+    /**
      * Indicates a null DocId. Will be returned if the root node is asked for
      * its parent.
      */
@@ -112,7 +123,12 @@ abstract class DocId {
      * @return a <code>DocId</code> based on a document number.
      */
     static DocId create(int docNumber) {
-        return new PlainDocId(docNumber);
+        if (docNumber < Short.MAX_VALUE) {
+            // use cached values for docNumbers up to 32k
+            return LOW_DOC_IDS[docNumber];
+        } else {
+            return new PlainDocId(docNumber);
+        }
     }
 
     /**

Added: jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java?rev=924677&view=auto
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java
(added)
+++ jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java
Thu Mar 18 09:48:03 2010
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.integration;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+
+import org.apache.jackrabbit.core.AbstractConcurrencyTest;
+
+/**
+ * <code>ConcurrentQueriesWithUpdatesTest</code> runs concurrent queries that
+ * stress the hierarchy cache in CachingIndexReader combined with updates that
+ * modify the hierarchy cache.
+ */
+public class ConcurrentQueriesWithUpdatesTest extends AbstractConcurrencyTest {
+
+    private static final int NUM_UPDATES = 50;
+
+    private int numNodes;
+
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+        numNodes = createNodes(testRootNode, 2, 12, 0); // ~4k nodes
+        superuser.save();
+    }
+
+    public void testQueriesWithUpdates() throws Exception {
+        final List<Exception> exceptions = Collections.synchronizedList(new ArrayList<Exception>());
+        final AtomicBoolean running = new AtomicBoolean(true);
+        // track executed queries and do updates at most at the given rate
+        final BlockingQueue<Object> queryExecuted = new LinkedBlockingQueue<Object>();
+        Thread queries = new Thread(new Runnable() {
+            public void run() {
+                try {
+                    runTask(new Task() {
+                        public void execute(Session session, Node test)
+                                throws RepositoryException {
+                            QueryManager qm = session.getWorkspace().getQueryManager();
+                            while (running.get()) {
+                                Query q = qm.createQuery(testPath + "//element(*, nt:unstructured)
order by @jcr:score descending", Query.XPATH);
+                                System.out.println(Thread.currentThread().getName() + " executed
query");
+                                NodeIterator nodes = q.execute().getNodes();
+                                assertEquals("wrong result set size", numNodes, nodes.getSize());
+                                queryExecuted.offer(new Object());
+                            }
+                        }
+                    }, 5, testRootNode.getPath());
+                } catch (RepositoryException e) {
+                    exceptions.add(e);
+                }
+            }
+        });
+        queries.start();
+        Thread update = new Thread(new Runnable() {
+            public void run() {
+                try {
+                    runTask(new Task() {
+                        public void execute(Session session, Node test)
+                                throws RepositoryException {
+                            Random rand = new Random();
+                            QueryManager qm = session.getWorkspace().getQueryManager();
+                            for (int i = 0; i < NUM_UPDATES; i++) {
+                                try {
+                                    // wait at most 10 seconds
+                                    queryExecuted.poll(10, TimeUnit.SECONDS);
+                                } catch (InterruptedException e) {
+                                    // ignore
+                                }
+                                Query q = qm.createQuery(testPath + "//node" + rand.nextInt(numNodes)
+ " order by @jcr:score descending", Query.XPATH);
+                                NodeIterator nodes = q.execute().getNodes();
+                                if (nodes.hasNext()) {
+                                    Node n = nodes.nextNode();
+                                    n.setProperty("foo", "bar");
+                                    System.out.println("updated " + n.getPath());
+                                    session.save();
+                                }
+                            }
+                        }
+                    }, 1, testRootNode.getPath());
+                } catch (RepositoryException e) {
+                    exceptions.add(e);
+                }
+            }
+        });
+        update.start();
+        update.join();
+        running.set(false);
+        queries.join();
+    }
+
+    private int createNodes(Node n, int nodesPerLevel, int levels, int count)
+            throws RepositoryException {
+        levels--;
+        for (int i = 0; i < nodesPerLevel; i++) {
+            Node child = n.addNode("node" + count);
+            count++;
+            if (count % 1000 == 0) {
+                superuser.save();
+            }
+            if (levels > 0) {
+                count = createNodes(child, nodesPerLevel, levels, count);
+            }
+        }
+        return count;
+    }
+}

Propchange: jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/ConcurrentQueriesWithUpdatesTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/daily/DailyIntegrationTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/daily/DailyIntegrationTest.java?rev=924677&r1=924676&r2=924677&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/daily/DailyIntegrationTest.java
(original)
+++ jackrabbit/trunk/jackrabbit-core/src/test/java/org/apache/jackrabbit/core/integration/daily/DailyIntegrationTest.java
Thu Mar 18 09:48:03 2010
@@ -29,6 +29,7 @@ import org.apache.jackrabbit.core.Concur
 import org.apache.jackrabbit.core.ConcurrentVersioningWithTransactionsTest;
 import org.apache.jackrabbit.core.LockTest;
 import org.apache.jackrabbit.core.ReadVersionsWhileModified;
+import org.apache.jackrabbit.core.integration.ConcurrentQueriesWithUpdatesTest;
 import org.apache.jackrabbit.core.query.LargeResultSetTest;
 import org.apache.jackrabbit.core.lock.ConcurrentLockingTest;
 import org.apache.jackrabbit.core.lock.ConcurrentLockingWithTransactionsTest;
@@ -58,6 +59,7 @@ public class DailyIntegrationTest extend
         suite.addTestSuite(ConcurrentLockingTest.class);
         suite.addTestSuite(ConcurrentLockingWithTransactionsTest.class);
         suite.addTestSuite(LargeResultSetTest.class);
+        suite.addTestSuite(ConcurrentQueriesWithUpdatesTest.class);
 
         return suite;
     }

Modified: jackrabbit/trunk/jackrabbit-spi-commons/src/main/java/org/apache/jackrabbit/spi/commons/name/HashCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/trunk/jackrabbit-spi-commons/src/main/java/org/apache/jackrabbit/spi/commons/name/HashCache.java?rev=924677&r1=924676&r2=924677&view=diff
==============================================================================
--- jackrabbit/trunk/jackrabbit-spi-commons/src/main/java/org/apache/jackrabbit/spi/commons/name/HashCache.java
(original)
+++ jackrabbit/trunk/jackrabbit-spi-commons/src/main/java/org/apache/jackrabbit/spi/commons/name/HashCache.java
Thu Mar 18 09:48:03 2010
@@ -28,18 +28,26 @@ package org.apache.jackrabbit.spi.common
 public class HashCache {
 
     /**
-     * Size of the cache (must be a power of two). Note that this is the
-     * maximum number of objects kept in the cache, but due to hashing it
-     * can well be that only a part of the cache array is filled even if
-     * many more distinct objects are being accessed.
+     * Array of cached objects, indexed by their hash codes
+     * (module size of the array).
      */
-    private static final int SIZE_POWER_OF_2 = 1024;
+    private final Object[] array;
 
     /**
-     * Array of cached objects, indexed by their hash codes
-     * (module size of the array).
+     * Creates a hash cache with 1024 slots.
+     */
+    public HashCache() {
+        this(10);
+    }
+
+    /**
+     * Creates a hash cache with 2^<code>exponent</code> slots.
+     *
+     * @param exponent the exponent.
      */
-    private final Object[] array = new Object[SIZE_POWER_OF_2];
+    public HashCache(int exponent) {
+        this.array = new Object[2 << exponent];
+    }
 
     /**
      * If a cached copy of the given object already exists, then returns
@@ -49,7 +57,7 @@ public class HashCache {
      * @return the given object or a previously cached copy
      */
     public Object get(Object object) {
-        int position = object.hashCode() & (SIZE_POWER_OF_2 - 1);
+        int position = object.hashCode() & (array.length - 1);
         Object previous = array[position];
         if (object.equals(previous)) {
             return previous;



Mime
View raw message