jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From resc...@apache.org
Subject svn commit: r1233399 - in /jackrabbit/branches/2.2: ./ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
Date Thu, 19 Jan 2012 14:29:59 GMT
Author: reschke
Date: Thu Jan 19 14:29:59 2012
New Revision: 1233399

URL: http://svn.apache.org/viewvc?rev=1233399&view=rev
Log:
JCR-3107: Speed up hierarchy cache initialization

(backported to 2.2)

Modified:
    jackrabbit/branches/2.2/   (props changed)
    jackrabbit/branches/2.2/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java

Propchange: jackrabbit/branches/2.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 19 14:29:59 2012
@@ -3,4 +3,4 @@
 /jackrabbit/sandbox/JCR-1456:774917-886178
 /jackrabbit/sandbox/JCR-2170:812417-816332
 /jackrabbit/sandbox/tripod-JCR-2209:795441-795863
-/jackrabbit/trunk:1038201,1038203,1038205,1038657,1039064,1039347,1039408,1039422-1039423,1039888,1039946,1040033,1040090,1040459,1040601,1040606,1040661,1040958,1041379,1041439,1041761,1042643,1042647,1042978-1042982,1043084-1043086,1043088,1043343,1043357-1043358,1043430,1043554,1043616,1043618,1043637,1043656,1043893,1043897,1044239,1044312,1044451,1044613,1049473,1049491,1049514,1049518,1049520,1049859,1049870,1049874,1049878,1049880,1049883,1049889,1049891,1049894-1049895,1049899-1049901,1049909-1049911,1049915-1049916,1049919,1049923,1049925,1049931,1049936,1049939,1050212,1050298,1050346,1050551,1055068,1055070-1055071,1055116-1055117,1055127,1055134,1055164,1055498,1060431,1060434,1060753,1063756,1064213,1064670,1065599,1065622,1066059,1066071,1066794,1069831,1071562,1071573,1071680,1072087,1074140,1077927,1077970,1079314,1079317,1080186,1080540,1082599,1082611,1082620,1087304,1088991,1089032,1089053,1089436,1092106,1092117,1092683,1097363,1097513-1097514,1098963-109
 8964,1099033,1099172,1100242,1100286,1101046,1102262,1102268-1102270,1102299,1102601,1104027,1126987,1128175,1129206,1130192,1130228,1132993,1136353,1136360,1138511,1141141,1141717,1143396,1143738,1144332,1144338,1144695,1152258,1155431,1157175,1165609,1173196,1174822,1174887,1175988,1176423,1176465,1176515,1176546,1177249,1177340,1178251,1178892,1179124,1179548,1180922,1181712,1182281,1182667,1182929,1183409,1185691,1186285,1186802,1187344,1188541,1188590,1198827
+/jackrabbit/trunk:1038201,1038203,1038205,1038657,1039064,1039347,1039408,1039422-1039423,1039888,1039946,1040033,1040090,1040459,1040601,1040606,1040661,1040958,1041379,1041439,1041761,1042643,1042647,1042978-1042982,1043084-1043086,1043088,1043343,1043357-1043358,1043430,1043554,1043616,1043618,1043637,1043656,1043893,1043897,1044239,1044312,1044451,1044613,1049473,1049491,1049514,1049518,1049520,1049859,1049870,1049874,1049878,1049880,1049883,1049889,1049891,1049894-1049895,1049899-1049901,1049909-1049911,1049915-1049916,1049919,1049923,1049925,1049931,1049936,1049939,1050212,1050298,1050346,1050551,1055068,1055070-1055071,1055116-1055117,1055127,1055134,1055164,1055498,1060431,1060434,1060753,1063756,1064213,1064670,1065599,1065622,1066059,1066071,1066794,1069831,1071562,1071573,1071680,1072087,1074140,1077927,1077970,1079314,1079317,1080186,1080540,1082599,1082611,1082620,1087304,1088991,1089032,1089053,1089436,1092106,1092117,1092683,1097363,1097513-1097514,1098963-109
 8964,1099033,1099172,1100242,1100286,1101046,1102262,1102268-1102270,1102299,1102601,1104027,1126987,1128175,1129206,1130192,1130228,1132993,1136353,1136360,1138511,1141141,1141717,1143396,1143738,1144332,1144338,1144695,1152258,1155431,1157175,1165609,1173196,1174822,1174887,1175988,1176423,1176465,1176515,1176546,1177249,1177340,1178251,1178892,1179124,1179548,1180922,1181712,1182281,1182667,1182761,1182824,1182929,1183409,1185691,1186285,1186802,1187344,1188541,1188590,1198827

Modified: jackrabbit/branches/2.2/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/branches/2.2/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java?rev=1233399&r1=1233398&r2=1233399&view=diff
==============================================================================
--- jackrabbit/branches/2.2/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
(original)
+++ jackrabbit/branches/2.2/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/CachingIndexReader.java
Thu Jan 19 14:29:59 2012
@@ -16,29 +16,32 @@
  */
 package org.apache.jackrabbit.core.query.lucene;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.text.NumberFormat;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.commons.collections.map.LRUMap;
+import org.apache.jackrabbit.core.id.NodeId;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FilterIndexReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermDocs;
 import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.jackrabbit.core.id.NodeId;
-import org.apache.commons.collections.map.LRUMap;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Collections;
-import java.text.NumberFormat;
-import java.util.concurrent.ConcurrentHashMap;
-
 /**
  * Implements an <code>IndexReader</code> that maintains caches to resolve
  * {@link #getParent(int, BitSet)} calls efficiently.
@@ -86,7 +89,7 @@ class CachingIndexReader extends FilterI
      * Initializes the {@link #inSegmentParents} and {@link #foreignParentDocIds}
      * caches.
      */
-    private CacheInitializer cacheInitializer;
+    private final CacheInitializer cacheInitializer;
 
     /**
      * Tick when this index reader was created.
@@ -128,16 +131,7 @@ class CachingIndexReader extends FilterI
         this.cache = cache;
         this.inSegmentParents = new int[delegatee.maxDoc()];
         Arrays.fill(this.inSegmentParents, -1);
-        this.shareableNodes = new BitSet();
-        TermDocs tDocs = delegatee.termDocs(
-                new Term(FieldNames.SHAREABLE_NODE, ""));
-        try {
-            while (tDocs.next()) {
-                shareableNodes.set(tDocs.doc());
-            }
-        } finally {
-            tDocs.close();
-        }
+        this.shareableNodes = initShareableNodes(delegatee);
         this.cacheInitializer = new CacheInitializer(delegatee);
         if (initCache) {
             cacheInitializer.run();
@@ -148,6 +142,20 @@ class CachingIndexReader extends FilterI
         this.termDocsCache = new TermDocsCache(delegatee, FieldNames.PROPERTIES);
     }
 
+    private BitSet initShareableNodes(IndexReader delegatee) throws IOException {
+        BitSet shareableNodes = new BitSet();
+        TermDocs tDocs = delegatee.termDocs(new Term(FieldNames.SHAREABLE_NODE,
+                ""));
+        try {
+            while (tDocs.next()) {
+                shareableNodes.set(tDocs.doc());
+            }
+        } finally {
+            tDocs.close();
+        }
+        return shareableNodes;
+    }
+
     /**
      * Returns the <code>DocId</code> of the parent of <code>n</code>
or
      * {@link DocId#NULL} if <code>n</code> does not have a parent
@@ -355,6 +363,11 @@ class CachingIndexReader extends FilterI
     private class CacheInitializer implements Runnable {
 
         /**
+         * The {@link #inSegmentParents} is persisted using this filename.
+         */
+        private static final String FILE_CACHE_NAME_ARRAY = "cache.inSegmentParents";
+
+        /**
          * From where to read.
          */
         private final IndexReader reader;
@@ -371,8 +384,8 @@ class CachingIndexReader extends FilterI
 
         /**
          * Creates a new initializer with the given <code>reader</code>.
-         *
-         * @param reader an index reader.
+         * @param reader
+         *            an index reader.
          */
         public CacheInitializer(IndexReader reader) {
             this.reader = reader;
@@ -390,7 +403,13 @@ class CachingIndexReader extends FilterI
                     // immediately return when stop is requested
                     return;
                 }
-                initializeParents(reader);
+                boolean initCacheFromFile = loadCacheFromFile();
+                if (!initCacheFromFile) {
+                    // file-based cache is not available, load from the
+                    // repository
+                    log.debug("persisted cache is not available, will load directly from
the repository.");
+                    initializeParents(reader);
+                }
             } catch (Exception e) {
                 // only log warn message during regular operation
                 if (!stopRequested) {
@@ -463,7 +482,8 @@ class CachingIndexReader extends FilterI
                 });
 
                 if (docs.isEmpty()) {
-                    // no more nodes to initialize
+                    // no more nodes to initialize, persist cache to file
+                    saveCacheToFile();
                     break;
                 }
 
@@ -591,6 +611,69 @@ class CachingIndexReader extends FilterI
                 tDocs.close();
             }
         }
+
+        /**
+         * Persists the cache info {@link #inSegmentParents} to a file:
+         * {@link #FILE_CACHE_NAME_ARRAY}, for faster init times on startup.
+         * 
+         * see https://issues.apache.org/jira/browse/JCR-3107
+         */
+        public void saveCacheToFile() throws IOException {
+            IndexOutput io = null;
+            try {
+                io = reader.directory().createOutput(FILE_CACHE_NAME_ARRAY);
+                for (int parent : inSegmentParents) {
+                    io.writeInt(parent);
+                }
+            } catch (Exception e) {
+                log.error(
+                        "Error saving " + FILE_CACHE_NAME_ARRAY + ": "
+                                + e.getMessage(), e);
+            } finally {
+                io.close();
+            }
+        }
+
+        /**
+         * Loads the cache info {@link #inSegmentParents} from the file
+         * {@link #FILE_CACHE_NAME_ARRAY}.
+         * 
+         * see https://issues.apache.org/jira/browse/JCR-3107
+         * 
+         * @return true if the cache has been initialized of false if the cache
+         *         file does not exist yet, or an error happened
+         */
+        private boolean loadCacheFromFile() throws IOException {
+            IndexInput ii = null;
+            try {
+                long time = System.currentTimeMillis();
+                ii = reader.directory().openInput(FILE_CACHE_NAME_ARRAY);
+                for (int i = 0; i < inSegmentParents.length; i++) {
+                    inSegmentParents[i] = ii.readInt();
+                }
+                log.debug(
+                        "persisted cache initialized {} DocIds in {} ms",
+                        new Object[] { inSegmentParents.length,
+                                System.currentTimeMillis() - time });
+                return true;
+            } catch (FileNotFoundException ignore) {
+                // expected in the case where the file-based cache has not been
+                // initialized yet
+            } catch (IOException ignore) {
+                log.warn(
+                        "Saved state of CachingIndexReader is corrupt, will try to remove
offending file "
+                                + FILE_CACHE_NAME_ARRAY, ignore);
+                // In the case where is a read error, the cache file is removed
+                // so it can be recreated after
+                // the cache loads the data from the repository directly
+                reader.directory().deleteFile(FILE_CACHE_NAME_ARRAY);
+            } finally {
+                if (ii != null) {
+                    ii.close();
+                }
+            }
+            return false;
+        }
     }
 
     /**



Mime
View raw message