lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1063510 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/ lucene/src/java/org/apache/lucene/index/ luce...
Date Tue, 25 Jan 2011 23:10:21 GMT
Author: mikemccand
Date: Tue Jan 25 23:10:20 2011
New Revision: 1063510

URL: http://svn.apache.org/viewvc?rev=1063510&view=rev
Log:
LUCENE-2474: add expert ReaderFinishedListener API, for external caches to evict entries for
readers

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/CHANGES.txt
    lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
    lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Jan 25 23:10:20 2011
@@ -488,6 +488,11 @@ New features
 
 * LUCENE-2720: Segments now record the code version which created them. 
   (Shai Erera, Mike McCandless, Uwe Schindler)
+
+* LUCENE-2474: Added expert ReaderFinishedListener API to
+  IndexReader, to allow apps that maintain external per-segment caches
+  to evict entries when a segment is finished.  (Shay Banon, Yonik
+  Seeley, Mike McCandless)
   
 Optimizations
 

Modified: lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
Tue Jan 25 23:10:20 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.store.instanti
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -46,6 +47,7 @@ public class InstantiatedIndexReader ext
   public InstantiatedIndexReader(InstantiatedIndex index) {
     super();
     this.index = index;
+    readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
   }
 
   /**

Modified: lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
Tue Jan 25 23:10:20 2011
@@ -25,6 +25,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 
@@ -735,6 +736,7 @@ public class MemoryIndex implements Seri
     
     private MemoryIndexReader() {
       super(); // avoid as much superclass baggage as possible
+      readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
     }
     
     private Info getInfo(String fieldName) {

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
Tue Jan 25 23:10:20 2011
@@ -36,8 +36,6 @@ import org.apache.lucene.search.Similari
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
-
 /** 
  * An IndexReader which reads indexes with multiple segments.
  */
@@ -75,20 +73,28 @@ class DirectoryReader extends IndexReade
         SegmentInfos infos = new SegmentInfos();
         infos.read(directory, segmentFileName);
         if (readOnly)
-          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+          return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor,
null);
         else
-          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+          return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor,
null);
       }
     }.run(commit);
   }
 
   /** Construct reading the named set of readers. */
-  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy,
boolean readOnly, int termInfosIndexDivisor) throws IOException {
+  DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy,
boolean readOnly, int termInfosIndexDivisor,
+                  Collection<ReaderFinishedListener> readerFinishedListeners) throws
IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = sis;
     this.deletionPolicy = deletionPolicy;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+
+    if (readerFinishedListeners == null) {
+      this.readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
+    } else {
+      this.readerFinishedListeners = readerFinishedListeners;
+    }
+
     // To reduce the chance of hitting FileNotFound
     // (and having to retry), we open segments in
     // reverse because IndexWriter merges & deletes
@@ -99,6 +105,7 @@ class DirectoryReader extends IndexReade
       boolean success = false;
       try {
         readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
+        readers[i].readerFinishedListeners = this.readerFinishedListeners;
         success = true;
       } finally {
         if (!success) {
@@ -123,6 +130,7 @@ class DirectoryReader extends IndexReade
     this.readOnly = true;
     segmentInfos = (SegmentInfos) infos.clone();// make sure we clone otherwise we share
mutable state with IW
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    readerFinishedListeners = writer.getReaderFinishedListeners();
 
     // IndexWriter synchronizes externally before calling
     // us, which ensures infos will not change; so there's
@@ -137,6 +145,7 @@ class DirectoryReader extends IndexReade
         final SegmentInfo info = infos.info(i);
         assert info.dir == dir;
         readers[i] = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor);
+        readers[i].readerFinishedListeners = readerFinishedListeners;
         success = true;
       } finally {
         if (!success) {
@@ -159,11 +168,14 @@ class DirectoryReader extends IndexReade
 
   /** This constructor is only used for {@link #reopen()} */
   DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[]
oldStarts,
-                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone,
int termInfosIndexDivisor) throws IOException {
+                  Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone,
int termInfosIndexDivisor,
+                  Collection<ReaderFinishedListener> readerFinishedListeners) throws
IOException {
     this.directory = directory;
     this.readOnly = readOnly;
     this.segmentInfos = infos;
     this.termInfosIndexDivisor = termInfosIndexDivisor;
+    assert readerFinishedListeners != null;
+    this.readerFinishedListeners = readerFinishedListeners;
 
     // we put the old SegmentReaders in a map, that allows us
     // to lookup a reader using its segment name
@@ -203,8 +215,10 @@ class DirectoryReader extends IndexReade
 
           // this is a new reader; in case we hit an exception we can close it safely
           newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor);
+          newReader.readerFinishedListeners = readerFinishedListeners;
         } else {
           newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly);
+          assert newReader.readerFinishedListeners == readerFinishedListeners;
         }
         if (newReader == newReaders[i]) {
           // this reader will be shared between the old and the new one,
@@ -344,6 +358,7 @@ class DirectoryReader extends IndexReade
       writeLock = null;
       hasChanges = false;
     }
+    assert newReader.readerFinishedListeners != null;
 
     return newReader;
   }
@@ -378,7 +393,9 @@ class DirectoryReader extends IndexReade
     // TODO: right now we *always* make a new reader; in
     // the future we could have write make some effort to
     // detect that no changes have occurred
-    return writer.getReader();
+    IndexReader reader = writer.getReader();
+    reader.readerFinishedListeners = readerFinishedListeners;
+    return reader;
   }
 
   private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException,
IOException {
@@ -446,9 +463,9 @@ class DirectoryReader extends IndexReade
   private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean
openReadOnly) throws CorruptIndexException, IOException {
     DirectoryReader reader;
     if (openReadOnly) {
-      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache,
doClone, termInfosIndexDivisor);
+      reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache,
doClone, termInfosIndexDivisor, readerFinishedListeners);
     } else {
-      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false,
doClone, termInfosIndexDivisor);
+      reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false,
doClone, termInfosIndexDivisor, readerFinishedListeners);
     }
     return reader;
   }
@@ -857,11 +874,6 @@ class DirectoryReader extends IndexReade
       }
     }
 
-    // NOTE: only needed in case someone had asked for
-    // FieldCache for top-level reader (which is generally
-    // not a good idea):
-    FieldCache.DEFAULT.purge(this);
-
     if (writer != null) {
       // Since we just closed, writer may now be able to
       // delete unused files:

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
Tue Jan 25 23:10:20 2011
@@ -20,11 +20,12 @@ package org.apache.lucene.index;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
+import java.util.HashSet;
+import java.util.Collections;
 
 /**  A <code>FilterIndexReader</code> contains another IndexReader, which it
  * uses as its basic source of data, possibly transforming the data along the
@@ -108,6 +109,7 @@ public class FilterIndexReader extends I
   public FilterIndexReader(IndexReader in) {
     super();
     this.in = in;
+    readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
   }
 
   @Override
@@ -244,11 +246,6 @@ public class FilterIndexReader extends I
   @Override
   protected void doClose() throws IOException {
     in.close();
-
-    // NOTE: only needed in case someone had asked for
-    // FieldCache for top-level reader (which is generally
-    // not a good idea):
-    FieldCache.DEFAULT.purge(this);
   }
 
 
@@ -305,4 +302,17 @@ public class FilterIndexReader extends I
     buffer.append(')');
     return buffer.toString();
   }
+
+  @Override
+  public void addReaderFinishedListener(ReaderFinishedListener listener) {
+    super.addReaderFinishedListener(listener);
+    in.addReaderFinishedListener(listener);
+  }
+
+  @Override
+  public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+    super.removeReaderFinishedListener(listener);
+    in.removeReaderFinishedListener(listener);
+  }
 }
+

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
Tue Jan 25 23:10:20 2011
@@ -28,6 +28,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.Closeable;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -77,6 +78,65 @@ import java.util.concurrent.atomic.Atomi
 public abstract class IndexReader implements Cloneable,Closeable {
 
   /**
+   * A custom listener that's invoked when the IndexReader
+   * is finished.
+   *
+   * <p>For a SegmentReader, this listener is called only
+   * once all SegmentReaders sharing the same core are
+   * closed.  At this point it is safe for apps to evict
+   * this reader from any caches keyed on {@link
+   * #getCoreCacheKey}.  This is the same interface that
+   * {@link FieldCache} uses, internally, to evict
+   * entries.</p>
+   *
+   * <p>For other readers, this listener is called when they
+   * are closed.</p>
+   *
+   * @lucene.experimental
+   */
+  public static interface ReaderFinishedListener {
+    public void finished(IndexReader reader);
+  }
+
+  // Impls must set this if they may call add/removeReaderFinishedListener:
+  protected volatile Collection<ReaderFinishedListener> readerFinishedListeners;
+
+  /** Expert: adds a {@link ReaderFinishedListener}.  The
+   * provided listener is also added to any sub-readers, if
+   * this is a composite reader.  Also, any reader reopened
+   * or cloned from this one will also copy the listeners at
+   * the time of reopen.
+   *
+   * @lucene.experimental */
+  public void addReaderFinishedListener(ReaderFinishedListener listener) {
+    readerFinishedListeners.add(listener);
+  }
+
+  /** Expert: remove a previously added {@link ReaderFinishedListener}.
+   *
+   * @lucene.experimental */
+  public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+    readerFinishedListeners.remove(listener);
+  }
+
+  protected void notifyReaderFinishedListeners() {
+    // Defensive (should never be null -- all impls must set
+    // this):
+    if (readerFinishedListeners != null) {
+
+      // Clone the set so that we don't have to sync on
+      // readerFinishedListeners while invoking them:
+      for(ReaderFinishedListener listener : new HashSet<ReaderFinishedListener>(readerFinishedListeners))
{
+        listener.finished(this);
+      }
+    }
+  }
+
+  protected void readerFinished() {
+    notifyReaderFinishedListeners();
+  }
+
+  /**
    * Constants describing field properties, for example used for
    * {@link IndexReader#getFieldNames(FieldOption)}.
    */
@@ -190,6 +250,7 @@ public abstract class IndexReader implem
           refCount.incrementAndGet();
         }
       }
+      readerFinished();
     }
   }
   

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
Tue Jan 25 23:10:20 2011
@@ -30,6 +30,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.Collections;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.Analyzer;
@@ -460,6 +461,13 @@ public class IndexWriter implements Clos
     return r;
   }
 
+  // Used for all SegmentReaders we open
+  private final Collection<IndexReader.ReaderFinishedListener> readerFinishedListeners
= Collections.synchronizedSet(new HashSet<IndexReader.ReaderFinishedListener>());
+
+  Collection<IndexReader.ReaderFinishedListener> getReaderFinishedListeners() throws
IOException {
+    return readerFinishedListeners;
+  }
+
   /** Holds shared SegmentReader instances. IndexWriter uses
    *  SegmentReaders for 1) applying deletes, 2) doing
    *  merges, 3) handing out a real-time reader.  This pool
@@ -669,6 +677,7 @@ public class IndexWriter implements Clos
         // synchronized
         // Returns a ref, which we xfer to readerMap:
         sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor);
+        sr.readerFinishedListeners = readerFinishedListeners;
 
         if (info.dir == directory) {
           // Only pool if reader is not external

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MultiReader.java
Tue Jan 25 23:10:20 2011
@@ -22,6 +22,8 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.HashSet;
+import java.util.Collections;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldSelector;
@@ -29,7 +31,6 @@ import org.apache.lucene.index.Directory
 import org.apache.lucene.index.DirectoryReader.MultiTermEnum;
 import org.apache.lucene.index.DirectoryReader.MultiTermPositions;
 import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
 
 /** An IndexReader which reads multiple indexes, appending
  * their content. */
@@ -84,6 +85,7 @@ public class MultiReader extends IndexRe
         hasDeletions = true;
     }
     starts[subReaders.length] = maxDoc;
+    readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
   }
   
   /**
@@ -416,11 +418,6 @@ public class MultiReader extends IndexRe
         subReaders[i].close();
       }
     }
-
-    // NOTE: only needed in case someone had asked for
-    // FieldCache for top-level reader (which is generally
-    // not a good idea):
-    FieldCache.DEFAULT.purge(this);
   }
   
   @Override
@@ -456,4 +453,20 @@ public class MultiReader extends IndexRe
   public IndexReader[] getSequentialSubReaders() {
     return subReaders;
   }
+
+  @Override
+  public void addReaderFinishedListener(ReaderFinishedListener listener) {
+    super.addReaderFinishedListener(listener);
+    for(IndexReader sub : subReaders) {
+      sub.addReaderFinishedListener(listener);
+    }
+  }
+
+  @Override
+  public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+    super.removeReaderFinishedListener(listener);
+    for(IndexReader sub : subReaders) {
+      sub.removeReaderFinishedListener(listener);
+    }
+  }
 }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ParallelReader.java
Tue Jan 25 23:10:20 2011
@@ -21,7 +21,6 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.FieldSelector;
 import org.apache.lucene.document.FieldSelectorResult;
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
 
 import java.io.IOException;
 import java.util.*;
@@ -68,6 +67,7 @@ public class ParallelReader extends Inde
   public ParallelReader(boolean closeSubReaders) throws IOException {
     super();
     this.incRefReaders = !closeSubReaders;
+    readerFinishedListeners = Collections.synchronizedSet(new HashSet<ReaderFinishedListener>());
   }
 
   /** {@inheritDoc} */
@@ -486,8 +486,6 @@ public class ParallelReader extends Inde
         readers.get(i).close();
       }
     }
-
-    FieldCache.DEFAULT.purge(this);
   }
 
   @Override
@@ -661,6 +659,21 @@ public class ParallelReader extends Inde
     }
   }
 
+  @Override
+  public void addReaderFinishedListener(ReaderFinishedListener listener) {
+    super.addReaderFinishedListener(listener);
+    for (IndexReader reader : readers) {
+      reader.addReaderFinishedListener(listener);
+    }
+  }
+
+  @Override
+  public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+    super.removeReaderFinishedListener(listener);
+    for (IndexReader reader : readers) {
+      reader.removeReaderFinishedListener(listener);
+    }
+  }
 }
 
 

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
Tue Jan 25 23:10:20 2011
@@ -21,15 +21,17 @@ import org.apache.lucene.store.Directory
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Collection;
 
 class ReadOnlyDirectoryReader extends DirectoryReader {
-  ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy,
int termInfosIndexDivisor) throws IOException {
-    super(directory, sis, deletionPolicy, true, termInfosIndexDivisor);
+  ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy,
int termInfosIndexDivisor,
+                          Collection<ReaderFinishedListener> readerFinishedListeners)
throws IOException {
+    super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, readerFinishedListeners);
   }
 
   ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders,
int[] oldStarts,  Map<String,byte[]> oldNormsCache, boolean doClone,
-                          int termInfosIndexDivisor) throws IOException {
-    super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor);
+                          int termInfosIndexDivisor, Collection<ReaderFinishedListener>
readerFinishedListeners) throws IOException {
+    super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor,
readerFinishedListeners);
   }
   
   ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor)
throws IOException {

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
Tue Jan 25 23:10:20 2011
@@ -38,7 +38,6 @@ import org.apache.lucene.store.IndexInpu
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.CloseableThreadLocal;
-import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache
entry on close
 
 /**
  * @lucene.experimental
@@ -242,13 +241,9 @@ public class SegmentReader extends Index
           storeCFSReader.close();
         }
 
-        // Force FieldCache to evict our entries at this
-        // point.  If the exception occurred while
-        // initializing the core readers, then
-        // origInstance will be null, and we don't want
-        // to call FieldCache.purge (it leads to NPE):
+        // Now, notify any ReaderFinished listeners:
         if (origInstance != null) {
-          FieldCache.DEFAULT.purge(origInstance);
+          origInstance.notifyReaderFinishedListeners();
         }
       }
     }
@@ -715,6 +710,7 @@ public class SegmentReader extends Index
       clone.si = si;
       clone.readBufferSize = readBufferSize;
       clone.pendingDeleteCount = pendingDeleteCount;
+      clone.readerFinishedListeners = readerFinishedListeners;
 
       if (!openReadOnly && hasChanges) {
         // My pending changes transfer to the new reader
@@ -1368,4 +1364,14 @@ public class SegmentReader extends Index
   public int getTermInfosIndexDivisor() {
     return core.termsIndexDivisor;
   }
+
+  @Override
+  protected void readerFinished() {
+    // Do nothing here -- we have more careful control on
+    // when to notify that a SegmentReader has finished,
+    // because a given core is shared across many cloned
+    // SegmentReaders.  We only notify once that core is no
+    // longer used (all SegmentReaders sharing it have been
+    // closed).
+  }
 }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
Tue Jan 25 23:10:20 2011
@@ -132,6 +132,13 @@ class FieldCacheImpl implements FieldCac
   static final class StopFillCacheException extends RuntimeException {
   }
 
+  final static IndexReader.ReaderFinishedListener purgeReader = new IndexReader.ReaderFinishedListener()
{
+    // @Override -- not until Java 1.6
+    public void finished(IndexReader reader) {
+      FieldCache.DEFAULT.purge(reader);
+    }
+  };
+
   /** Expert: Internal cache. */
   abstract static class Cache {
     Cache() {
@@ -164,8 +171,10 @@ class FieldCacheImpl implements FieldCac
       synchronized (readerCache) {
         innerCache = readerCache.get(readerKey);
         if (innerCache == null) {
+          // First time this reader is using FieldCache
           innerCache = new HashMap<Entry,Object>();
           readerCache.put(readerKey, innerCache);
+          reader.addReaderFinishedListener(purgeReader);
           value = null;
         } else {
           value = innerCache.get(key);

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063510&r1=1063509&r2=1063510&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
Tue Jan 25 23:10:20 2011
@@ -1790,4 +1790,42 @@ public class TestIndexReader extends Luc
     assertTrue(IndexReader.indexExists(dir));
     dir.close();
   }
+
+  // LUCENE-2474
+  public void testReaderFinishedListener() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+    ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3);
+    writer.setInfoStream(VERBOSE ? System.out : null);
+    writer.addDocument(new Document());
+    writer.commit();
+    writer.addDocument(new Document());
+    writer.commit();
+    final IndexReader reader = writer.getReader();
+    final int[] closeCount = new int[1];
+    final IndexReader.ReaderFinishedListener listener = new IndexReader.ReaderFinishedListener()
{
+      public void finished(IndexReader reader) {
+        closeCount[0]++;
+      }
+    };
+
+    reader.addReaderFinishedListener(listener);
+
+    reader.close();
+
+    // Just the top reader
+    assertEquals(1, closeCount[0]);
+    writer.close();
+
+    // Now also the subs
+    assertEquals(3, closeCount[0]);
+
+    IndexReader reader2 = IndexReader.open(dir);
+    reader2.addReaderFinishedListener(listener);
+
+    closeCount[0] = 0;
+    reader2.close();
+    assertEquals(3, closeCount[0]);
+    dir.close();
+  }
 }



Mime
View raw message