lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r945556 - in /lucene/java/branches/lucene_3_0: ./ contrib/ contrib/highlighter/src/test/ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/index/ src/java/org...
Date Tue, 18 May 2010 09:32:07 GMT
Author: mikemccand
Date: Tue May 18 09:32:07 2010
New Revision: 945556

URL: http://svn.apache.org/viewvc?rev=945556&view=rev
Log:
LUCENE-2467: fix memory leaks in IW when indexing very large docs

Modified:
    lucene/java/branches/lucene_3_0/   (props changed)
    lucene/java/branches/lucene_3_0/CHANGES.txt   (contents, props changed)
    lucene/java/branches/lucene_3_0/build.xml   (props changed)
    lucene/java/branches/lucene_3_0/contrib/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/CHANGES.txt   (props changed)
    lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java   (props
changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
  (props changed)

Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene:932398
+/lucene/dev/trunk/lucene:932398,945420
 /lucene/java/branches/lucene_2_4:748824
 /lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336

Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Tue May 18 09:32:07 2010
@@ -19,6 +19,9 @@ Bug fixes
   potentially excessive memory usage when many threads are indexing a
   mix of small and large documents.  (Tim Smith via Mike McCandless)
   
+* LUCENE-2467: Fixed memory leaks in IndexWriter when large documents
+  are indexed.  (Mike McCandless)
+
 API Changes
 
 * LUCENE-2281: added doBeforeFlush to IndexWriter to allow extensions to perform

Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/CHANGES.txt:932398
+/lucene/dev/trunk/lucene/CHANGES.txt:932398,945420
 /lucene/java/branches/lucene_2_4/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/build.xml:932398
+/lucene/dev/trunk/lucene/build.xml:932398,945420
 /lucene/java/branches/lucene_2_4/build.xml:748824
 /lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib:932398
+/lucene/dev/trunk/lucene/contrib:932398,945420
 /lucene/java/branches/lucene_2_4/contrib:748824
 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/CHANGES.txt:932398
+/lucene/dev/trunk/lucene/contrib/CHANGES.txt:932398,945420
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/highlighter/src/test:932398
+/lucene/dev/trunk/lucene/contrib/highlighter/src/test:932398,945420
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:932398
+/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:932398,945420
 /lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
 /lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:932398
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DocumentsWriter.java
Tue May 18 09:32:07 2010
@@ -1260,11 +1260,14 @@ final class DocumentsWriter {
     }
 
     /* Return byte[]'s to the pool */
+
     @Override
     void recycleByteBlocks(byte[][] blocks, int start, int end) {
       synchronized(DocumentsWriter.this) {
-        for(int i=start;i<end;i++)
+        for(int i=start;i<end;i++) {
           freeByteBlocks.add(blocks[i]);
+          blocks[i] = null;
+        }
       }
     }
   }
@@ -1300,7 +1303,6 @@ final class DocumentsWriter {
 
   synchronized void bytesAllocated(long numBytes) {
     numBytesAlloc += numBytes;
-    assert numBytesUsed <= numBytesAlloc;
   }
 
   synchronized void bytesUsed(long numBytes) {
@@ -1310,8 +1312,10 @@ final class DocumentsWriter {
 
   /* Return int[]s to the pool */
   synchronized void recycleIntBlocks(int[][] blocks, int start, int end) {
-    for(int i=start;i<end;i++)
+    for(int i=start;i<end;i++) {
       freeIntBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
   }
 
   ByteBlockAllocator byteBlockAllocator = new ByteBlockAllocator(BYTE_BLOCK_SIZE);
@@ -1351,8 +1355,10 @@ final class DocumentsWriter {
 
   /* Return char[]s to the pool */
   synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
-    for(int i=0;i<numBlocks;i++)
+    for(int i=0;i<numBlocks;i++) {
       freeCharBlocks.add(blocks[i]);
+      blocks[i] = null;
+    }
   }
 
   String toMB(long v) {
@@ -1412,7 +1418,7 @@ final class DocumentsWriter {
             // Nothing else to free -- must flush now.
             bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
             if (infoStream != null) {
-              if (numBytesUsed > flushTrigger)
+              if (bufferIsFull)
                 message("    nothing to free; now set bufferIsFull");
               else
                 message("    nothing to free");

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java Tue
May 18 09:32:07 2010
@@ -3512,7 +3512,13 @@ public class IndexWriter implements Clos
   // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
     try {
-      return doFlushInternal(flushDocStores, flushDeletes);
+      try {
+        return doFlushInternal(flushDocStores, flushDeletes);
+      } finally {
+        if (docWriter.doBalanceRAM()) {
+          docWriter.balanceRAM();
+        }
+      }
     } finally {
       docWriter.clearFlushPending();
     }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/ReusableStringReader.java
Tue May 18 09:32:07 2010
@@ -43,6 +43,8 @@ final class ReusableStringReader extends
       left -= len;
       return len;
     } else if (0 == left) {
+      // don't keep a reference (s could have been very large)
+      s = null;
       return -1;
     } else {
       s.getChars(upto, upto+left, c, off);

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
Tue May 18 09:32:07 2010
@@ -193,7 +193,6 @@ final class TermVectorsTermsWriter exten
         lastPos = pos;
       }
       perDoc.perDocTvf.writeTo(tvf);
-      perDoc.perDocTvf.reset();
       perDoc.numVectorFields = 0;
     }
 
@@ -201,6 +200,7 @@ final class TermVectorsTermsWriter exten
 
     lastDocID++;
 
+    perDoc.reset();
     free(perDoc);
     assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument end");
   }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java?rev=945556&r1=945555&r2=945556&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/TermsHash.java Tue May
18 09:32:07 2010
@@ -89,8 +89,17 @@ final class TermsHash extends InvertedDo
 
     assert postingsFreeCount == postingsAllocCount: Thread.currentThread().getName() + ":
postingsFreeCount=" + postingsFreeCount + " postingsAllocCount=" + postingsAllocCount + "
consumer=" + consumer;
 
-    final int newSize = ArrayUtil.getShrinkSize(postingsFreeList.length, postingsAllocCount);
+    final int newSize = 1;
     if (newSize != postingsFreeList.length) {
+      if (postingsFreeCount > newSize) {
+        if (trackAllocations) {
+          docWriter.bytesAllocated(-(postingsFreeCount-newSize) * bytesPerPosting);
+        }
+
+        postingsFreeCount = newSize;
+        postingsAllocCount = newSize;
+      }
+
       RawPostingList[] newArray = new RawPostingList[newSize];
       System.arraycopy(postingsFreeList, 0, newArray, 0, postingsFreeCount);
       postingsFreeList = newArray;

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:932398
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,888247,889431-889432,890967,890988,891189,891209,891363,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 18 09:32:07 2010
@@ -1,4 +1,4 @@
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:932398
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:932398,945420
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336



Mime
View raw message