lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r826426 - /lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
Date Sun, 18 Oct 2009 13:03:13 GMT
Author: mikemccand
Date: Sun Oct 18 13:03:12 2009
New Revision: 826426

URL: http://svn.apache.org/viewvc?rev=826426&view=rev
Log:
LUCENE-1458: reduce lower bound memory usage of standard codec's terms dict index

Modified:
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java?rev=826426&r1=826425&r2=826426&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
Sun Oct 18 13:03:12 2009
@@ -115,6 +115,7 @@
     } finally {
       if (indexDivisor != -1) {
         in.close();
+        trimByteBlock();
         indexLoaded = true;
         this.in = null;
       } else {
@@ -124,6 +125,35 @@
     }
   }
 
+  /* Called when index is fully loaded.  We know we will use
+   * no more bytes in the final byte[], so trim it down to
+   * its actual usagee.  This substantially reduces memory
+   * usage of SegmentReader searching a tiny segment. */
+  private final void trimByteBlock() {
+    if (blockUpto != 0) {
+      if (blockOffset == 0) {
+        // nocommit -- should not happen?  fields w/ no terms
+        // are not written by STDW.  hmmm it does
+        // happen... must explain why
+        // assert false;
+        blocks[blockUpto] = null;
+      } else {
+        byte[] last = new byte[blockOffset];
+        System.arraycopy(blocks[blockUpto], 0, last, 0, blockOffset);
+        blocks[blockUpto] = last;
+      }
+    } else {
+      // nocommit -- we shouldn't get here, but we do -- fix it!
+    }
+  }
+
+  // nocommit -- we can record precisely how many bytes are
+  // required during indexing, save that into file, and be
+  // precise when we allocate the blocks; we even don't need
+  // to use blocks anymore (though my still want to, to
+  // prevent allocation failure due to mem fragmentation on
+  // 32bit)
+
   // Fixed size byte blocks, to hold all term bytes; these
   // blocks are shared across fields
   private byte[][] blocks;
@@ -432,6 +462,7 @@
         it.next().loadTermsIndex();
       }
       indexLoaded = true;
+      trimByteBlock();
     }
   }
 



Mime
View raw message