jackrabbit-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ju...@apache.org
Subject svn commit: r734187 - in /jackrabbit/branches/1.5: ./ jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/
Date Tue, 13 Jan 2009 17:13:48 GMT
Author: jukka
Date: Tue Jan 13 09:13:37 2009
New Revision: 734187

URL: http://svn.apache.org/viewvc?rev=734187&view=rev
Log:
1.5: Merged revision 734092 (JCR-1931)

Modified:
    jackrabbit/branches/1.5/   (props changed)
    jackrabbit/branches/1.5/RELEASE-NOTES.txt
    jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java
    jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java

Propchange: jackrabbit/branches/1.5/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 13 09:13:37 2009
@@ -1,2 +1,2 @@
 /jackrabbit/branches/1.3:631261
-/jackrabbit/trunk:703899-704158,704165,704167,704324,704358,704361,704864,704933,704939,705010,705033,705243,705496,705522,705579,705925,705932,705934,705937-705938,705961,706242,706273,706285-706286,706562,706606,706649,706655,706660,706697,706918,707303-707304,707307,707310,707630,708206,708598,708609,708613,708619,708634,708840,708863,708909,708929,708943,709115,709142,709207,709211,710047,711238,711566-711567,711595,711841-711843,712984-712985,713037,713059,713065,713072,713076,713162,713214,713956,713958,713964,713971,713975,714034,718218,718249-718250,718371,718376,718566,718632,718981,719225,719282,719575-719577,719579,719585-719586,719588,719592,720455,720484,720492,720524,720533,720540,720673,720679,720687,720784,720940,720969,721186,721191,721194,721235,721387-721389,721470,721495,722068-722069,722463,722465,722467,722470,722825,723281,723346,723728,723784,724300,724387,725292,727376,727388,727390,727395,727397,727402,727492,727701,728022,731896,731934,731941,73234
 7,732678,732686,732689,732693,732703,732715-732716,732719,732728,732730,732734-732738,732740,732742-732743,732745,732867,732883,733057-733059,733061,733080
+/jackrabbit/trunk:703899-704158,704165,704167,704324,704358,704361,704864,704933,704939,705010,705033,705243,705496,705522,705579,705925,705932,705934,705937-705938,705961,706242,706273,706285-706286,706562,706606,706649,706655,706660,706697,706918,707303-707304,707307,707310,707630,708206,708598,708609,708613,708619,708634,708840,708863,708909,708929,708943,709115,709142,709207,709211,710047,711238,711566-711567,711595,711841-711843,712984-712985,713037,713059,713065,713072,713076,713162,713214,713956,713958,713964,713971,713975,714034,718218,718249-718250,718371,718376,718566,718632,718981,719225,719282,719575-719577,719579,719585-719586,719588,719592,720455,720484,720492,720524,720533,720540,720673,720679,720687,720784,720940,720969,721186,721191,721194,721235,721387-721389,721470,721495,722068-722069,722463,722465,722467,722470,722825,723281,723346,723728,723784,724300,724387,725292,727376,727388,727390,727395,727397,727402,727492,727701,728022,731896,731934,731941,73234
 7,732678,732686,732689,732693,732703,732715-732716,732719,732728,732730,732734-732738,732740,732742-732743,732745,732867,732883,733057-733059,733061,733080,734092

Modified: jackrabbit/branches/1.5/RELEASE-NOTES.txt
URL: http://svn.apache.org/viewvc/jackrabbit/branches/1.5/RELEASE-NOTES.txt?rev=734187&r1=734186&r2=734187&view=diff
==============================================================================
--- jackrabbit/branches/1.5/RELEASE-NOTES.txt (original)
+++ jackrabbit/branches/1.5/RELEASE-NOTES.txt Tue Jan 13 09:13:37 2009
@@ -43,6 +43,7 @@
   [JCR-1823] Repository.login throws IllegalStateException
   [JCR-1838] Garbage collection deletes temporary files in FileDataStore
   [JCR-1920] Custom LoginModule configurations broken in 1.5.0
+  [JCR-1931] SharedFieldCache$StringIndex memory leak causing OOM's
 
 jackrabbit-jcr-commons
 

Modified: jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java?rev=734187&r1=734186&r2=734187&view=diff
==============================================================================
--- jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java
(original)
+++ jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java
Tue Jan 13 09:13:37 2009
@@ -43,6 +43,13 @@
     public static class StringIndex {
 
         /**
+         * Some heuristic factor that determines whether the array is sparse. Note that if
less then
+         * 1% is set, we already count the array as sparse. This is because it will become
memory consuming
+         * quickly by keeping the (sparse) arrays 
+         */
+        private static final int SPARSE_FACTOR = 100;
+
+        /**
          * All the term values, in natural order.
          */
         public final String[] lookup;
@@ -50,15 +57,65 @@
         /**
          * Terms indexed by document id.
          */
-        public final String[] terms;
+        private final String[] terms;
+
+        /**
+         * Terms map indexed by document id.
+         */
+        public final Map termsMap;
+
+        /**
+         * Boolean indicating whether the hashMap impl has to be used
+         */
+        public final boolean sparse;
 
         /**
          * Creates one of these objects
          */
-        public StringIndex(String[] terms, String[] lookup) {
-            this.terms = terms;
+        public StringIndex(String[] terms, String[] lookup, int setValues) {
+            if (isSparse(terms, setValues)) {
+                this.sparse = true;
+                this.terms = null;
+                if (setValues == 0) {
+                    this.termsMap = null;
+                } else {
+                    this.termsMap = getTermsMap(terms, setValues);
+                }
+            } else {
+                this.sparse = false;
+                this.terms = terms;
+                this.termsMap = null;
+            }
             this.lookup = lookup;
         }
+
+        public String getTerm(int i) {
+            if (sparse) {
+                return termsMap == null ? null : (String) termsMap.get(new Integer(i));
+            } else {
+                return terms[i];
+            }
+        }
+
+        private Map getTermsMap(String[] terms, int setValues) {
+            Map map = new HashMap(setValues);
+            for (int i = 0; i < terms.length && setValues > 0; i++) {
+                if (terms[i] != null) {
+                    map.put(new Integer(i), terms[i]);
+                    setValues--;
+                }
+            }
+            return map;
+        }
+
+        private boolean isSparse(String[] terms, int setValues) {
+            // some really simple test to test whether the array is sparse. Currently, when
less then 1% is set, the array is already sparse 
+            // for this typical cache to avoid memory issues
+            if (setValues * SPARSE_FACTOR < terms.length) {
+                return true;
+            }
+            return false;
+        }
     }
 
     /**
@@ -114,6 +171,7 @@
             if (includeLookup) {
                 mterms = new ArrayList();
             }
+            int setValues = 0;
             if (retArray.length > 0) {
                 TermDocs termDocs = reader.termDocs();
                 TermEnum termEnum = reader.terms(new Term(field, prefix));
@@ -141,6 +199,7 @@
 
                         termDocs.seek(termEnum);
                         while (termDocs.next()) {
+                            setValues++;
                             retArray[termDocs.doc()] = term.text().substring(prefix.length());
                         }
                     } while (termEnum.next());
@@ -153,7 +212,7 @@
             if (includeLookup) {
                 lookup = (String[]) mterms.toArray(new String[mterms.size()]);
             }
-            SharedFieldCache.StringIndex value = new SharedFieldCache.StringIndex(retArray,
lookup);
+            SharedFieldCache.StringIndex value = new SharedFieldCache.StringIndex(retArray,
lookup, setValues);
             store(reader, field, prefix, comparator, value);
             return value;
         }

Modified: jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java
URL: http://svn.apache.org/viewvc/jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java?rev=734187&r1=734186&r2=734187&view=diff
==============================================================================
--- jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java
(original)
+++ jackrabbit/branches/1.5/jackrabbit-core/src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java
Tue Jan 13 09:13:37 2009
@@ -117,8 +117,8 @@
                 int idx1 = readerIndex(i.doc);
                 int idx2 = readerIndex(j.doc);
 
-                String iTerm = indexes[idx1].terms[i.doc - starts[idx1]];
-                String jTerm = indexes[idx2].terms[j.doc - starts[idx2]];
+                String iTerm = indexes[idx1].getTerm(i.doc - starts[idx1]);
+                String jTerm = indexes[idx2].getTerm(j.doc - starts[idx2]);
 
                 if (iTerm == jTerm) {
                     return 0;
@@ -141,8 +141,8 @@
              */
             public Comparable sortValue(final ScoreDoc i) {
                 if (createComparatorValues) {
-                    StringIndex index = indexes[readerIndex(i.doc)];
-                    return index.terms[i.doc];
+                    int idx = readerIndex(i.doc);
+                    return indexes[idx].getTerm(i.doc - starts[idx]);
                 } else {
                     // return dummy value
                     return "";



Mime
View raw message