lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r950580 - in /lucene/java/branches/lucene_3_0: ./ contrib/ contrib/analyzers/ contrib/highlighter/src/test/ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/...
Date Wed, 02 Jun 2010 14:37:33 GMT
Author: mikemccand
Date: Wed Jun  2 14:37:32 2010
New Revision: 950580

URL: http://svn.apache.org/viewvc?rev=950580&view=rev
Log:
LUCENE-2468: allow specifying how CachingWrapperFilter should handle deletions on reopened
readers

Added:
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
      - copied, changed from r949288, lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
Modified:
    lucene/java/branches/lucene_3_0/   (props changed)
    lucene/java/branches/lucene_3_0/CHANGES.txt   (contents, props changed)
    lucene/java/branches/lucene_3_0/build.xml   (props changed)
    lucene/java/branches/lucene_3_0/contrib/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/CHANGES.txt   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java   (props
changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/FilterIndexReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingSpanFilter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingWrapperFilter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
  (props changed)

Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene:941394,946651,948430
-/lucene/dev/trunk/lucene:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4:748824
 /lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336

Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Wed Jun  2 14:37:32 2010
@@ -65,6 +65,18 @@ Bug fixes
 * LUCENE-2478: Fix CachingWrapperFilter to not throw NPE when
   Filter.getDocIdSet() returns null.  (Uwe Schindler, Daniel Noll)
 
+* LUCENE-2468: Allow specifying how new deletions should be handled in
+  CachingWrapperFilter and CachingSpanFilter.  By default, new
+  deletions are ignored in CachingWrapperFilter, since typically this
+  filter is AND'd with a query that correctly takes new deletions into
+  account.  This should be a performance gain (higher cache hit rate)
+  in apps that reopen readers, or use near-real-time reader
+  (IndexWriter.getReader()), but may introduce invalid search results
+  (allowing deleted docs to be returned) for certain cases, so a new
+  expert ctor was added to CachingWrapperFilter to enforce deletions
+  at a performance cost.  CachingSpanFilter by default recaches if
+  there are new deletions (Shay Banon via Mike McCandless)
+
 * LUCENE-2299: If you open an NRT reader while addIndexes* is running,
   it may miss some segments (Earwin Burrfoot via Mike McCandless)
 

Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/CHANGES.txt:941394,946651,948430
-/lucene/dev/trunk/lucene/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/build.xml:941394,946651,948430
-/lucene/dev/trunk/lucene/build.xml:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/build.xml:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/build.xml:748824
 /lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/contrib:941394,946651,948430
-/lucene/dev/trunk/lucene/contrib:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/contrib:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/contrib:748824
 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:941394,946651,948430
-/lucene/dev/trunk/lucene/contrib/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/contrib/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,3 +1,3 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/analyzers:941394,946651
-/lucene/dev/trunk/lucene/contrib/analyzers:929738,935522,939611,940730,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/contrib/analyzers:929738,935522,939611,940730,948429,949288,949976,949997,950458
 /lucene/java/trunk/contrib/analyzers:887670,889579,889866,890439,891205,891377,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:941394,946651,948430
-/lucene/dev/trunk/lucene/contrib/highlighter/src/test:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/contrib/highlighter/src/test:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:941394,946651,948430
-/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
 /lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/FilterIndexReader.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/FilterIndexReader.java
Wed Jun  2 14:37:32 2010
@@ -288,4 +288,12 @@ public class FilterIndexReader extends I
   public Object getFieldCacheKey() {
     return in.getFieldCacheKey();
   }
+
+  /** If the subclass of FilteredIndexReader modifies the
+   *  deleted docs, you must override this method to provide
+   *  a different key */
+  @Override
+  public Object getDeletesCacheKey() {
+    return in.getDeletesCacheKey();
+  }
 }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexReader.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexReader.java Wed
Jun  2 14:37:32 2010
@@ -1149,6 +1149,12 @@ public abstract class IndexReader implem
     return this;
   }
 
+  /** Expert.  Warning: this returns null if the reader has
+   *  no deletions */
+  public Object getDeletesCacheKey() {
+    return this;
+  }
+
   /** Returns the number of unique terms (across all fields)
    *  in this reader.
    *

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java Wed
Jun  2 14:37:32 2010
@@ -1278,6 +1278,11 @@ public class SegmentReader extends Index
   }
 
   @Override
+  public Object getDeletesCacheKey() {
+    return deletedDocs;
+  }
+
+  @Override
   public long getUniqueTermCount() {
     return core.getTermsReader().size();
   }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingSpanFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingSpanFilter.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingSpanFilter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingSpanFilter.java
Wed Jun  2 14:37:32 2010
@@ -19,9 +19,6 @@ package org.apache.lucene.search;
 import org.apache.lucene.index.IndexReader;
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.WeakHashMap;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Wraps another SpanFilter's result and caches it.  The purpose is to allow
@@ -33,15 +30,32 @@ public class CachingSpanFilter extends S
   /**
    * A transient Filter cache (package private because of test)
    */
-  private transient Map<IndexReader,SpanFilterResult> cache;
-
-  private final ReentrantLock lock = new ReentrantLock();
+  private final CachingWrapperFilter.FilterCache<SpanFilterResult> cache;
 
   /**
+   * New deletions always result in a cache miss, by default
+   * ({@link CachingWrapperFilter.DeletesMode#RECACHE}.
    * @param filter Filter to cache results of
    */
   public CachingSpanFilter(SpanFilter filter) {
+    this(filter, CachingWrapperFilter.DeletesMode.RECACHE);
+  }
+
+  /**
+   * @param filter Filter to cache results of
+   * @param deletesMode See {@link CachingWrapperFilter.DeletesMode}
+   */
+  public CachingSpanFilter(SpanFilter filter, CachingWrapperFilter.DeletesMode deletesMode)
{
     this.filter = filter;
+    if (deletesMode == CachingWrapperFilter.DeletesMode.DYNAMIC) {
+      throw new IllegalArgumentException("DeletesMode.DYNAMIC is not supported");
+    }
+    this.cache = new CachingWrapperFilter.FilterCache<SpanFilterResult>(deletesMode)
{
+      @Override
+      protected SpanFilterResult mergeDeletes(final IndexReader r, final SpanFilterResult
value) {
+        throw new IllegalStateException("DeletesMode.DYNAMIC is not supported");
+      }
+    };
   }
 
   @Override
@@ -50,26 +64,24 @@ public class CachingSpanFilter extends S
     return result != null ? result.getDocIdSet() : null;
   }
   
+  // for testing
+  int hitCount, missCount;
+
   private SpanFilterResult getCachedResult(IndexReader reader) throws IOException {
-    lock.lock();
-    try {
-      if (cache == null) {
-        cache = new WeakHashMap<IndexReader,SpanFilterResult>();
-      }
-      final SpanFilterResult cached = cache.get(reader);
-      if (cached != null) return cached;
-    } finally {
-      lock.unlock();
-    }
-    
-    final SpanFilterResult result = filter.bitSpans(reader);
-    lock.lock();
-    try {
-      cache.put(reader, result);
-    } finally {
-      lock.unlock();
+
+    final Object coreKey = reader.getFieldCacheKey();
+    final Object delCoreKey = reader.hasDeletions() ? reader.getDeletesCacheKey() : coreKey;
+
+    SpanFilterResult result = cache.get(reader, coreKey, delCoreKey);
+    if (result != null) {
+      hitCount++;
+      return result;
     }
-    
+
+    missCount++;
+    result = filter.bitSpans(reader);
+
+    cache.put(coreKey, delCoreKey, result);
     return result;
   }
 

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingWrapperFilter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/CachingWrapperFilter.java
Wed Jun  2 14:37:32 2010
@@ -17,10 +17,10 @@ package org.apache.lucene.search;
  * limitations under the License.
  */
 
+import java.io.Serializable;
 import java.io.IOException;
 import java.util.Map;
 import java.util.WeakHashMap;
-import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.util.OpenBitSetDISI;
@@ -33,17 +33,127 @@ public class CachingWrapperFilter extend
   Filter filter;
 
   /**
-   * A transient Filter cache (package private because of test)
+   * Expert: Specifies how new deletions against a reopened
+   * reader should be handled.
+   *
+   * <p>The default is IGNORE, which means the cache entry
+   * will be re-used for a given segment, even when that
+   * segment has been reopened due to changes in deletions.
+   * This is a big performance gain, especially with
+   * near-real-timer readers, since you don't hit a cache
+   * miss on every reopened reader for prior segments.</p>
+   *
+   * <p>However, in some cases this can cause invalid query
+   * results, allowing deleted documents to be returned.
+   * This only happens if the main query does not rule out
+   * deleted documents on its own, such as a toplevel
+   * ConstantScoreQuery.  To fix this, use RECACHE to
+   * re-create the cached filter (at a higher per-reopen
+   * cost, but at faster subsequent search performance), or
+   * use DYNAMIC to dynamically intersect deleted docs (fast
+   * reopen time but some hit to search performance).</p>
    */
-  transient Map<IndexReader, DocIdSet> cache;
-  
-  private final ReentrantLock lock = new ReentrantLock();
+  public static enum DeletesMode {IGNORE, RECACHE, DYNAMIC};
+
+  protected final FilterCache<DocIdSet> cache;
+
+  static abstract class FilterCache<T> implements Serializable {
+
+    /**
+     * A transient Filter cache (package private because of test)
+     */
+    // NOTE: not final so that we can dynamically re-init
+    // after de-serialize
+    transient Map<Object,T> cache;
+
+    private final DeletesMode deletesMode;
+
+    public FilterCache(DeletesMode deletesMode) {
+      this.deletesMode = deletesMode;
+    }
+
+    public synchronized T get(IndexReader reader, Object coreKey, Object delCoreKey) throws
IOException {
+      T value;
+
+      if (cache == null) {
+        cache = new WeakHashMap<Object,T>();
+      }
+
+      if (deletesMode == DeletesMode.IGNORE) {
+        // key on core
+        value = cache.get(coreKey);
+      } else if (deletesMode == DeletesMode.RECACHE) {
+        // key on deletes, if any, else core
+        value = cache.get(delCoreKey);
+      } else {
+
+        assert deletesMode == DeletesMode.DYNAMIC;
+
+        // first try for exact match
+        value = cache.get(delCoreKey);
+
+        if (value == null) {
+          // now for core match, but dynamically AND NOT
+          // deletions
+          value = cache.get(coreKey);
+          if (value != null && reader.hasDeletions()) {
+            value = mergeDeletes(reader, value);
+          }
+        }
+      }
+
+      return value;
+    }
+
+    protected abstract T mergeDeletes(IndexReader reader, T value);
+
+    public synchronized void put(Object coreKey, Object delCoreKey, T value) {
+      if (deletesMode == DeletesMode.IGNORE) {
+        cache.put(coreKey, value);
+      } else if (deletesMode == DeletesMode.RECACHE) {
+        cache.put(delCoreKey, value);
+      } else {
+        cache.put(coreKey, value);
+        cache.put(delCoreKey, value);
+      }
+    }
+  }
 
   /**
+   * New deletes are ignored by default, which gives higher
+   * cache hit rate on reopened readers.  Most of the time
+   * this is safe, because the filter will be AND'd with a
+   * Query that fully enforces deletions.  If instead you
+   * need this filter to always enforce deletions, pass
+   * either {@link DeletesMode#RECACHE} or {@link
+   * DeletesMode#DYNAMIC}.
    * @param filter Filter to cache results of
    */
   public CachingWrapperFilter(Filter filter) {
+    this(filter, DeletesMode.IGNORE);
+  }
+
+  /**
+   * Expert: by default, the cached filter will be shared
+   * across reopened segments that only had changes to their
+   * deletions.  
+   *
+   * @param filter Filter to cache results of
+   * @param deletesMode See {@link DeletesMode}
+   */
+  public CachingWrapperFilter(Filter filter, DeletesMode deletesMode) {
     this.filter = filter;
+    cache = new FilterCache<DocIdSet>(deletesMode) {
+      @Override
+      public DocIdSet mergeDeletes(final IndexReader r, final DocIdSet docIdSet) {
+        return new FilteredDocIdSet(docIdSet) {
+          @Override
+          protected boolean match(int docID) {
+            return !r.isDeleted(docID);
+          }
+        };
+      }
+    };
   }
 
   /** Provide the DocIdSet to be cached, using the DocIdSet provided
@@ -66,29 +176,29 @@ public class CachingWrapperFilter extend
       return (it == null) ? DocIdSet.EMPTY_DOCIDSET : new OpenBitSetDISI(it, reader.maxDoc());
     }
   }
-  
+
+  // for testing
+  int hitCount, missCount;
+
   @Override
   public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-    lock.lock();
-    try {
-      if (cache == null) {
-        cache = new WeakHashMap<IndexReader,DocIdSet>();
-      }
 
-      final DocIdSet cached = cache.get(reader);
-      if (cached != null) return cached;
-    } finally {
-      lock.unlock();
+    final Object coreKey = reader.getFieldCacheKey();
+    final Object delCoreKey = reader.hasDeletions() ? reader.getDeletesCacheKey() : coreKey;
+
+    DocIdSet docIdSet = cache.get(reader, coreKey, delCoreKey);
+    if (docIdSet != null) {
+      hitCount++;
+      return docIdSet;
     }
 
-    final DocIdSet docIdSet = docIdSetToCache(filter.getDocIdSet(reader), reader);
+    missCount++;
+
+    // cache miss
+    docIdSet = docIdSetToCache(filter.getDocIdSet(reader), reader);
+
     if (docIdSet != null) {
-      lock.lock();
-      try {
-        cache.put(reader, docIdSet);
-      } finally {
-        lock.unlock();
-      }
+      cache.put(coreKey, delCoreKey, docIdSet);
     }
     
     return docIdSet;

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,3 +1,3 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index:941394,946651
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:929738,935522,940730,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:929738,935522,940730,948429,949288,949976,949997,950458
 /lucene/java/trunk/src/test/org/apache/lucene/index:887670,889579,889866,890439,891205,891377,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Modified: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java
(original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java
Wed Jun  2 14:37:32 2010
@@ -18,10 +18,8 @@ package org.apache.lucene.search;
  */
 
 import java.io.IOException;
-import java.util.BitSet;
-import java.util.WeakHashMap;
 
-import junit.framework.TestCase;
+import junit.framework.Assert;
 
 import org.apache.lucene.index.IndexReader;
 
@@ -44,30 +42,18 @@ public class CachingWrapperFilterHelper 
   }
   
   @Override
-  public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-    if (cache == null) {
-      cache = new WeakHashMap();
-    }
-    
-    synchronized (cache) {  // check cache
-      DocIdSet cached = (DocIdSet) cache.get(reader);
-      if (shouldHaveCache) {
-        TestCase.assertNotNull("Cache should have data ", cached);
-      } else {
-        TestCase.assertNull("Cache should be null " + cached , cached);
-      }
-      if (cached != null) {
-        return cached;
-      }
-    }
+  public synchronized DocIdSet getDocIdSet(IndexReader reader) throws IOException {
 
-    final DocIdSet bits = filter.getDocIdSet(reader);
+    final int saveMissCount = missCount;
+    DocIdSet docIdSet = super.getDocIdSet(reader);
 
-    synchronized (cache) {  // update cache
-      cache.put(reader, bits);
+    if (shouldHaveCache) {
+      Assert.assertEquals("Cache should have data ", saveMissCount, missCount);
+    } else {
+      Assert.assertTrue("Cache should be null " + docIdSet, missCount > saveMissCount);
     }
 
-    return bits;
+    return docIdSet;
   }
 
   @Override

Copied: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
(from r949288, lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java)
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingSpanFilter.java?p2=lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingSpanFilter.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java&r1=949288&r2=950580&rev=950580&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingSpanFilter.java (original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingSpanFilter.java
Wed Jun  2 14:37:32 2010
@@ -19,10 +19,9 @@ package org.apache.lucene.search;
 
 import java.io.IOException;
 
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.document.Document;
@@ -35,7 +34,7 @@ public class TestCachingSpanFilter exten
 
   public void testEnforceDeletions() throws Exception {
     Directory dir = new MockRAMDirectory();
-    IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer()));
+    IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
     IndexReader reader = writer.getReader();
     IndexSearcher searcher = new IndexSearcher(reader);
 

Modified: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java?rev=950580&r1=950579&r2=950580&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
Wed Jun  2 14:37:32 2010
@@ -20,11 +20,15 @@ package org.apache.lucene.search;
 import java.io.IOException;
 
 import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.OpenBitSet;
 import org.apache.lucene.util.OpenBitSetDISI;
@@ -139,4 +143,123 @@ public class TestCachingWrapperFilter ex
 
     reader.close();
   }
+
+  public void testEnforceDeletions() throws Exception {
+    Directory dir = new MockRAMDirectory();
+    IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
+    IndexReader reader = writer.getReader();
+    IndexSearcher searcher = new IndexSearcher(reader);
+
+    // add a doc, refresh the reader, and check that its there
+    Document doc = new Document();
+    doc.add(new Field("id", "1", Field.Store.YES, Field.Index.NOT_ANALYZED));
+    writer.addDocument(doc);
+
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+
+    TopDocs docs = searcher.search(new MatchAllDocsQuery(), 1);
+    assertEquals("Should find a hit...", 1, docs.totalHits);
+
+    final Filter startFilter = new QueryWrapperFilter(new TermQuery(new Term("id", "1")));
+
+    // ignore deletions
+    CachingWrapperFilter filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.IGNORE);
+        
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
+    ConstantScoreQuery constantScore = new ConstantScoreQuery(filter);
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+    // now delete the doc, refresh the reader, and see that it's not there
+    writer.deleteDocuments(new Term("id", "1"));
+
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+
+    // force cache to regenerate:
+    filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.RECACHE);
+
+    writer.addDocument(doc);
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+        
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
+
+    constantScore = new ConstantScoreQuery(filter);
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+    // make sure we get a cache hit when we reopen reader
+    // that had no change to deletions
+    IndexReader newReader = refreshReader(reader);
+    assertTrue(reader != newReader);
+    reader = newReader;
+    searcher = new IndexSearcher(reader);
+    int missCount = filter.missCount;
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+    assertEquals(missCount, filter.missCount);
+
+    // now delete the doc, refresh the reader, and see that it's not there
+    writer.deleteDocuments(new Term("id", "1"));
+
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+
+    missCount = filter.missCount;
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals(missCount+1, filter.missCount);
+    assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
+
+
+    // apply deletions dynamically
+    filter = new CachingWrapperFilter(startFilter, CachingWrapperFilter.DeletesMode.DYNAMIC);
+
+    writer.addDocument(doc);
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+        
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals("[query + filter] Should find a hit...", 1, docs.totalHits);
+    constantScore = new ConstantScoreQuery(filter);
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should find a hit...", 1, docs.totalHits);
+
+    // now delete the doc, refresh the reader, and see that it's not there
+    writer.deleteDocuments(new Term("id", "1"));
+
+    reader = refreshReader(reader);
+    searcher = new IndexSearcher(reader);
+
+    docs = searcher.search(new MatchAllDocsQuery(), filter, 1);
+    assertEquals("[query + filter] Should *not* find a hit...", 0, docs.totalHits);
+
+    missCount = filter.missCount;
+    docs = searcher.search(constantScore, 1);
+    assertEquals("[just filter] Should *not* find a hit...", 0, docs.totalHits);
+
+    // doesn't count as a miss
+    assertEquals(missCount, filter.missCount);
+  }
+
+  private static IndexReader refreshReader(IndexReader reader) throws IOException {
+    IndexReader oldReader = reader;
+    reader = reader.reopen();
+    if (reader != oldReader) {
+      oldReader.close();
+    }
+    return reader;
+  }
 }

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jun  2 14:37:32 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:941394,946651,948430
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949976,949997,950458
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336



Mime
View raw message