lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r891377 - in /lucene/java/trunk: ./ src/java/org/apache/lucene/index/ src/java/org/apache/lucene/util/
Date Wed, 16 Dec 2009 18:51:45 GMT
Author: mikemccand
Date: Wed Dec 16 18:51:44 2009
New Revision: 891377

URL: http://svn.apache.org/viewvc?rev=891377&view=rev
Log:
LUCENE-2161: improve concurrency of IndexReader

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java
    lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/trunk/src/java/org/apache/lucene/util/BitVector.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=891377&r1=891376&r2=891377&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Wed Dec 16 18:51:44 2009
@@ -109,6 +109,9 @@
   the FieldCache rather than waiting for the WeakHashMap to release
   the reference (Mike McCandless)
 
+* LUCENE-2161: Improve concurrency of IndexReader, especially in the
+  context of near real-time readers.  (Mike McCandless)
+
 Build
 
  * LUCENE-2124: Moved the JDK-based collation support from contrib/collation 

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java?rev=891377&r1=891376&r2=891377&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java Wed Dec 16 18:51:44
2009
@@ -125,7 +125,7 @@
   DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws
IOException {
     this.directory = writer.getDirectory();
     this.readOnly = true;
-    this.segmentInfos = infos;
+    segmentInfos = infos;
     segmentInfosStart = (SegmentInfos) infos.clone();
     this.termInfosIndexDivisor = termInfosIndexDivisor;
     if (!readOnly) {
@@ -345,22 +345,39 @@
   }
 
   @Override
-  public final synchronized IndexReader reopen() throws CorruptIndexException, IOException
{
+  public final IndexReader reopen() throws CorruptIndexException, IOException {
     // Preserve current readOnly
     return doReopen(readOnly, null);
   }
 
   @Override
-  public final synchronized IndexReader reopen(boolean openReadOnly) throws CorruptIndexException,
IOException {
+  public final IndexReader reopen(boolean openReadOnly) throws CorruptIndexException, IOException
{
     return doReopen(openReadOnly, null);
   }
 
   @Override
-  public final synchronized IndexReader reopen(final IndexCommit commit) throws CorruptIndexException,
IOException {
+  public final IndexReader reopen(final IndexCommit commit) throws CorruptIndexException,
IOException {
     return doReopen(true, commit);
   }
 
-  private synchronized IndexReader doReopen(final boolean openReadOnly, IndexCommit commit)
throws CorruptIndexException, IOException {
+  private final IndexReader doReopenFromWriter(boolean openReadOnly, IndexCommit commit)
throws CorruptIndexException, IOException {
+    assert readOnly;
+
+    if (!openReadOnly) {
+      throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader()
can only be reopened with openReadOnly=true (got false)");
+    }
+
+    if (commit != null) {
+      throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader()
cannot currently accept a commit");
+    }
+
+    // TODO: right now we *always* make a new reader; in
+    // the future we could have write make some effort to
+    // detect that no changes have occurred
+    return writer.getReader();
+  }
+
+  private IndexReader doReopen(final boolean openReadOnly, IndexCommit commit) throws CorruptIndexException,
IOException {
     ensureOpen();
 
     assert commit == null || openReadOnly;
@@ -368,22 +385,13 @@
     // If we were obtained by writer.getReader(), re-ask the
     // writer to get a new reader.
     if (writer != null) {
-      assert readOnly;
-
-      if (!openReadOnly) {
-        throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader()
can only be reopened with openReadOnly=true (got false)");
-      }
-
-      if (commit != null) {
-        throw new IllegalArgumentException("a reader obtained from IndexWriter.getReader()
cannot currently accept a commit");
-      }
-
-      // TODO: right now we *always* make a new reader; in
-      // the future we could have write make some effort to
-      // detect that no changes have occurred
-      IndexReader reader = writer.getReader();
-      return reader;
+      return doReopenFromWriter(openReadOnly, commit);
+    } else {
+      return doReopenNoWriter(openReadOnly, commit);
     }
+  }
+
+  private synchronized IndexReader doReopenNoWriter(final boolean openReadOnly, IndexCommit
commit) throws CorruptIndexException, IOException {
 
     if (commit == null) {
       if (hasChanges) {
@@ -487,10 +495,13 @@
     ensureOpen();
     return segmentInfos.size() == 1 && !hasDeletions();
   }
-  
+
   @Override
-  public synchronized int numDocs() {
+  public int numDocs() {
     // Don't call ensureOpen() here (it could affect performance)
+
+    // NOTE: multiple threads may wind up init'ing
+    // numDocs... but that's harmless
     if (numDocs == -1) {        // check cache
       int n = 0;                // cache miss--recompute
       for (int i = 0; i < subReaders.length; i++)

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java?rev=891377&r1=891376&r2=891377&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/MultiReader.java Wed Dec 16 18:51:44
2009
@@ -224,8 +224,10 @@
   }
   
   @Override
-  public synchronized int numDocs() {
+  public int numDocs() {
     // Don't call ensureOpen() here (it could affect performance)
+    // NOTE: multiple threads may wind up init'ing
+    // numDocs... but that's harmless
     if (numDocs == -1) {        // check cache
       int n = 0;                // cache miss--recompute
       for (int i = 0; i < subReaders.length; i++)

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java?rev=891377&r1=891376&r2=891377&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java Wed Dec 16 18:51:44
2009
@@ -588,20 +588,28 @@
     core.openDocStores(si);
   }
 
+  private boolean checkDeletedCounts() throws IOException {
+    final int recomputedCount = deletedDocs.getRecomputedCount();
+     
+    assert deletedDocs.count() == recomputedCount : "deleted count=" + deletedDocs.count()
+ " vs recomputed count=" + recomputedCount;
+
+    assert si.getDelCount() == recomputedCount : 
+    "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + recomputedCount;
+
+    // Verify # deletes does not exceed maxDoc for this
+    // segment:
+    assert si.getDelCount() <= maxDoc() : 
+    "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for
segment " + si.name;
+
+    return true;
+  }
+
   private void loadDeletedDocs() throws IOException {
     // NOTE: the bitvector is stored using the regular directory, not cfs
     if (hasDeletions(si)) {
       deletedDocs = new BitVector(directory(), si.getDelFileName());
       deletedDocsRef = new AtomicInteger(1);
-     
-      assert si.getDelCount() == deletedDocs.count() : 
-        "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
-
-      // Verify # deletes does not exceed maxDoc for this
-      // segment:
-      assert si.getDelCount() <= maxDoc() : 
-        "delete count mismatch: " + deletedDocs.count() + ") exceeds max doc (" + maxDoc()
+ ") for segment " + si.name;
-
+      assert checkDeletedCounts();
     } else
       assert si.getDelCount() == 0;
   }

Modified: lucene/java/trunk/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/BitVector.java?rev=891377&r1=891376&r2=891377&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/BitVector.java Wed Dec 16 18:51:44 2009
@@ -36,24 +36,28 @@
 
   private byte[] bits;
   private int size;
-  private int count = -1;
+  private int count;
 
   /** Constructs a vector capable of holding <code>n</code> bits. */
   public BitVector(int n) {
     size = n;
     bits = new byte[(size >> 3) + 1];
+    count = 0;
   }
   
   BitVector(byte[] bits, int size) {
     this.bits = bits;
     this.size = size;
+    count = -1;
   }
   
   @Override
   public Object clone() {
     byte[] copyBits = new byte[bits.length];
     System.arraycopy(bits, 0, copyBits, 0, bits.length);
-    return new BitVector(copyBits, size);
+    BitVector clone = new BitVector(copyBits, size);
+    clone.count = count;
+    return clone;
   }
   
   /** Sets the value of <code>bit</code> to one. */
@@ -121,6 +125,15 @@
     return count;
   }
 
+  /** For testing */
+  public final int getRecomputedCount() {
+    int c = 0;
+    int end = bits.length;
+    for (int i = 0; i < end; i++)
+      c += BYTE_COUNTS[bits[i] & 0xFF];	  // sum bits per byte
+    return c;
+  }
+
   private static final byte[] BYTE_COUNTS = {	  // table of bits/byte
     0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
     1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,



Mime
View raw message