lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r945608 - in /lucene/java/branches/lucene_2_9: ./ src/java/org/apache/lucene/index/
Date Tue, 18 May 2010 12:12:03 GMT
Author: mikemccand
Date: Tue May 18 12:12:03 2010
New Revision: 945608

URL: http://svn.apache.org/viewvc?rev=945608&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs

Modified:
    lucene/java/branches/lucene_2_9/CHANGES.txt
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java

Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Tue May 18 12:12:03 2010
@@ -20,6 +20,9 @@ Bug fixes
    indexing a mix of small and large documents.  (Tim Smith via Mike
    McCandless)
   
+ * LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+   are indexed.  (Mike McCandless)
+
 ======================= Release 2.9.2 2010-02-26 =======================
 
 Bug fixes

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
(original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
Tue May 18 12:12:03 2010
@@ -1276,8 +1276,13 @@ final class DocumentsWriter {
     /* Return byte[]'s to the pool */
     void recycleByteBlocks(byte[][] blocks, int start, int end) {
       synchronized(DocumentsWriter.this) {
-        for(int i=start;i<end;i++)
+        for(int i=start;i<end;i++) {
           freeByteBlocks.add(blocks[i]);
+          blocks[i] = null;
+        }
+        if (infoStream != null && blockSize != 1024) {
+          message("DW.recycleByteBlocks blockSize=" + blockSize + " count=" + (end-start)
+ " total now " + freeByteBlocks.size());
+        }
       }
     }
   }
@@ -1313,7 +1318,6 @@ final class DocumentsWriter {
 
   synchronized void bytesAllocated(long numBytes) {
     numBytesAlloc += numBytes;
-    assert numBytesUsed <= numBytesAlloc;
   }
 
   synchronized void bytesUsed(long numBytes) {
@@ -1323,8 +1327,13 @@ final class DocumentsWriter {
 
   /* Return int[]s to the pool */
   synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
-    for(int i=start;i<end;i++)
+    for(int i=start;i<end;i++) {
       freeIntBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
+    if (infoStream != null) {
+      message("DW.recycleIntBlocks count=" + (end-start) + " total now " + freeIntBlocks.size());
+    }
   }
 
   ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
@@ -1364,8 +1373,13 @@ final class DocumentsWriter {
 
   /* Return char[]s to the pool */
   synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
-    for(int i=0;i<numBlocks;i++)
+    for(int i=0;i<numBlocks;i++) {
       freeCharBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
+    if (infoStream != null) {
+      message("DW.recycleCharBlocks count=" + numBlocks + " total now " + freeCharBlocks.size());
+    }
   }
 
   String toMB(long v) {
@@ -1425,7 +1439,7 @@ final class DocumentsWriter {
             // Nothing else to free -- must flush now.
             bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
             if (infoStream != null) {
-              if (numBytesUsed > flushTrigger)
+              if (bufferIsFull)
                 message("    nothing to free; now set bufferIsFull");
               else
                 message("    nothing to free");

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java Tue
May 18 12:12:03 2010
@@ -4208,6 +4208,9 @@ public class IndexWriter {
     try {
       return doFlushInternal(flushDocStores, flushDeletes);
     } finally {
+      if (docWriter.doBalanceRAM()) {
+        docWriter.balanceRAM();
+      }
       docWriter.clearFlushPending();
     }
   }

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
(original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/ReusableStringReader.java
Tue May 18 12:12:03 2010
@@ -41,6 +41,7 @@ final class ReusableStringReader extends
       left -= len;
       return len;
     } else if (0 == left) {
+      s = null;
       return -1;
     } else {
       s.getChars(upto, upto+left, c, off);

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
(original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
Tue May 18 12:12:03 2010
@@ -192,7 +192,6 @@ final class TermVectorsTermsWriter exten
         lastPos = pos;
       }
       perDoc.perDocTvf.writeTo(tvf);
-      perDoc.perDocTvf.reset();
       perDoc.numVectorFields = 0;
     }
 
@@ -200,6 +199,7 @@ final class TermVectorsTermsWriter exten
 
     lastDocID++;
 
+    perDoc.reset();
     free(perDoc);
     assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
   }

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java?rev=945608&r1=945607&r2=945608&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHash.java Tue May
18 12:12:03 2010
@@ -89,8 +89,17 @@ final class TermsHash extends InvertedDo
 
     assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ":
postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + "
consumer=" + consumer;
 
-    final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+    final int newSize = 1;
+
     if (newSize != postingsFreeList.length) {
+      if (postingsFreeCount > newSize) {
+        if (trackAllocations) {
+          docWriter.bytesAllocated(-(postingsFreeCount-newSize) * bytesPerPosting);
+        }
+        postingsFreeCount = newSize;
+        postingsAllocCount = newSize;
+      }
+    
       RawPostingList[] newArray = new RawPostingList[newSize];
       System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
       postingsFreeList = newArray;



Mime
View raw message