lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r948542 - in /lucene/dev/trunk/lucene: ./ contrib/ contrib/highlighter/src/test/ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/index/ src/java/org/apache/...
Date Wed, 26 May 2010 19:07:45 GMT
Author: mikemccand
Date: Wed May 26 19:07:44 2010
New Revision: 948542

URL: http://svn.apache.org/viewvc?rev=948542&view=rev
Log:
LUCENE-2467: more mem reduction during indexing

Modified:
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/build.xml   (props changed)
    lucene/dev/trunk/lucene/contrib/   (props changed)
    lucene/dev/trunk/lucene/contrib/CHANGES.txt   (props changed)
    lucene/dev/trunk/lucene/contrib/highlighter/src/test/   (props changed)
    lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
  (props changed)
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java   (props changed)
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
  (props changed)
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java   (props
changed)
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java   (props
changed)
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java   (props
changed)

Propchange: lucene/dev/trunk/lucene/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,7 +1,7 @@
 /lucene/dev/branches/branch_3x/lucene:943137
 /lucene/java/branches/flex_1458:824912-931101
 /lucene/java/branches/lucene_2_4:748824
-/lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9:817269-818600,829134,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
 /lucene/java/branches/lucene_3_0:880793,896906
 /lucene/java/trunk:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,4 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/build.xml:943137
 /lucene/java/branches/flex_1458/build.xml:824912-931101
-/lucene/java/branches/lucene_2_9/build.xml:909334
+/lucene/java/branches/lucene_2_9/build.xml:909334,948516
 /lucene/java/trunk/build.xml:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,7 +1,7 @@
 /lucene/dev/branches/branch_3x/lucene/contrib:943137
 /lucene/java/branches/flex_1458/contrib:824912-931101
 /lucene/java/branches/lucene_2_4/contrib:748824
-/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
 /lucene/java/branches/lucene_3_0/contrib:880793,896906
 /lucene/java/trunk/contrib:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,7 +1,7 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:943137
 /lucene/java/branches/flex_1458/contrib/CHANGES.txt:824912-931101
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850
+/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/CHANGES.txt:880793,896906
 /lucene/java/trunk/contrib/CHANGES.txt:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,7 +1,7 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:943137
 /lucene/java/branches/flex_1458/contrib/highlighter/src/test:824912-931101
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
-/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/highlighter/src/test:880793,896906
 /lucene/java/trunk/contrib/highlighter/src/test:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,4 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:943137
 /lucene/java/branches/flex_1458/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:824912-931101
-/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:909334,948516
 /lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,4 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:943137
 /lucene/java/branches/flex_1458/src/java/org/apache/lucene/analysis/Tokenizer.java:824912-931101
-/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334
+/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:909334,948516
 /lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:924483-924731,924781,925176-925462

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=948542&r1=948541&r2=948542&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Wed May
26 19:07:44 2010
@@ -162,6 +162,13 @@ final class DocumentsWriter {
     public boolean testPoint(String name) {
       return docWriter.writer.testPoint(name);
     }
+
+    public void clear() {
+      // don't hold onto doc nor analyzer, in case it is
+      // largish:
+      doc = null;
+      analyzer = null;
+    }
   }
 
   /** Consumer returns this on each doc.  This holds any
@@ -824,10 +831,16 @@ final class DocumentsWriter {
     try {
       // This call is not synchronized and does all the
       // work
-      final DocWriter perDoc = state.consumer.processDocument();
-        
+      final DocWriter perDoc;
+      try {
+        perDoc = state.consumer.processDocument();
+      } finally {
+        docState.clear();
+      }
+
       // This call is synchronized but fast
       finishDocument(state, perDoc);
+
       success = true;
     } finally {
       if (!success) {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=948542&r1=948541&r2=948542&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed May 26 19:07:44
2010
@@ -3672,6 +3672,9 @@ public class IndexWriter implements Clos
     // Make sure no threads are actively adding a document.
     // Returns true if docWriter is currently aborting, in
     // which case we skip flushing this segment
+    if (infoStream != null) {
+      message("flush: now pause all indexing threads");
+    }
     if (docWriter.pauseAllThreads()) {
       docWriter.resumeAllThreads();
       return false;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=948542&r1=948541&r2=948542&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Wed May
26 19:07:44 2010
@@ -98,13 +98,7 @@ final class TermsHashPerField extends In
   void shrinkHash(int targetSize) {
     assert postingsCompacted || numPostings == 0;
 
-    // Cannot use ArrayUtil.shrink because we require power
-    // of 2:
-    int newSize = postingsHash.length;
-    while(newSize >= 8 && newSize/4 > targetSize) {
-      newSize /= 2;
-    }
-
+    final int newSize = 4;
     if (newSize != postingsHash.length) {
       final long previousSize = postingsHash.length;
       postingsHash = new int[newSize];

Propchange: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,4 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137
 /lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:824912-931101
-/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334
+/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334,948516
 /lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:880793,896906
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:880793,896906
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334,948516
 /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:880793,896906,928290
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:924483-924731,924781,925176-925462

Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed May 26 19:07:44 2010
@@ -1,4 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:943137
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/util/TestAttributeSource.java:824912-931101
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:909334,948516
 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:924483-924731,924781,925176-925462



Mime
View raw message