lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r945420 - in /lucene/dev/trunk/lucene: ./ src/java/org/apache/lucene/index/
Date Mon, 17 May 2010 23:12:34 GMT
Author: mikemccand
Date: Mon May 17 23:12:34 2010
New Revision: 945420

URL: http://svn.apache.org/viewvc?rev=945420&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon May 17 23:12:34 2010
@@ -391,6 +391,9 @@ Bug fixes
   addition, all of IndexComnmit methods which threw 
   UnsupportedOperationException are now abstract. (Shai Erera)
 
+* LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+  are indexed.  (Mike McCandless)
+
 New features
 
 * LUCENE-2128: Parallelized fetching document frequencies during weight

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Mon May
17 23:12:34 2010
@@ -1314,11 +1314,14 @@ final class DocumentsWriter {
     }
 
     /* Return byte[]'s to the pool */
+
     @Override
     void recycleByteBlocks(byte[][] blocks, int start, int end) {
       synchronized(DocumentsWriter.this) {
-        for(int i=start;i<end;i++)
+        for(int i=start;i<end;i++) {
           freeByteBlocks.add(blocks[i]);
+          blocks[i] = null;
+        }
       }
     }
 
@@ -1326,9 +1329,11 @@ final class DocumentsWriter {
     void recycleByteBlocks(List<byte[]> blocks) {
       synchronized(DocumentsWriter.this) {
         final int size = blocks.size();
-        for(int i=0;i<size;i++)
+        for(int i=0;i<size;i++) {
           freeByteBlocks.add(blocks.get(i));
-  }
+          blocks.set(i, null);
+        }
+      }
     }
   }
 
@@ -1358,8 +1363,10 @@ final class DocumentsWriter {
 
   /* Return int[]s to the pool */
   synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
-    for(int i=start;i<end;i++)
+    for(int i=start;i<end;i++) {
       freeIntBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
   }
 
   ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon May 17 23:12:34
2010
@@ -3624,7 +3624,11 @@ public class IndexWriter implements Clos
   // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
     try {
-      return doFlushInternal(flushDocStores, flushDeletes);
+      try {
+        return doFlushInternal(flushDocStores, flushDeletes);
+      } finally {
+        docWriter.balanceRAM();
+      }
     } finally {
       docWriter.clearFlushPending();
     }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ReusableStringReader.java Mon
May 17 23:12:34 2010
@@ -43,6 +43,8 @@ final class ReusableStringReader extends
       left -= len;
       return len;
     } else if (0 == left) {
+      // don't keep a reference (s could have been very large)
+      s = null;
       return -1;
     } else {
       s.getChars(upto, upto+left, c, off);

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945420&r1=945419&r2=945420&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Mon
May 17 23:12:34 2010
@@ -190,7 +190,6 @@ final class TermVectorsTermsWriter exten
         lastPos = pos;
       }
       perDoc.perDocTvf.writeTo(tvf);
-      perDoc.perDocTvf.reset();
       perDoc.numVectorFields = 0;
     }
 
@@ -198,6 +197,7 @@ final class TermVectorsTermsWriter exten
 
     lastDocID++;
 
+    perDoc.reset();
     free(perDoc);
     assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
   }



Mime
View raw message