lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sim...@apache.org
Subject svn commit: r1336313 - /lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
Date Wed, 09 May 2012 17:41:20 GMT
Author: simonw
Date: Wed May  9 17:41:20 2012
New Revision: 1336313

URL: http://svn.apache.org/viewvc?rev=1336313&view=rev
Log:
LUCENE-4022: Offline Sorter wrongly uses MIN_BUFFER_SIZE if there is more memory available

Modified:
    lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java

Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java?rev=1336313&r1=1336312&r2=1336313&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
(original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
Wed May  9 17:41:20 2012
@@ -80,29 +80,35 @@ public final class Sort {
   
     /** 
      * Approximately half of the currently available free heap, but no less
-     * than {@link #MIN_BUFFER_SIZE_MB}. However if current heap allocation 
-     * is insufficient for sorting consult with max allowed heap size. 
+     * than {@link #ABSOLUTE_MIN_SORT_BUFFER_SIZE}. However if current heap allocation 
+     * is insufficient or if there is a large portion of unallocated heap-space available

+     * for sorting consult with max allowed heap size. 
      */
     public static BufferSize automatic() {
       Runtime rt = Runtime.getRuntime();
       
       // take sizes in "conservative" order
-      long max = rt.maxMemory();
-      long total = rt.totalMemory();
-      long free = rt.freeMemory();
-
+      final long max = rt.maxMemory(); // max allocated
+      final long total = rt.totalMemory(); // currently allocated
+      final long free = rt.freeMemory(); // unused portion of currently allocated
+      final long totalAvailableBytes = max - total + free;
+      
       // by free mem (attempting to not grow the heap for this)
-      long half = free/2;
-      if (half >= ABSOLUTE_MIN_SORT_BUFFER_SIZE) { 
-        return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+      long sortBufferByteSize = free/2;
+      final long minBufferSizeBytes = MIN_BUFFER_SIZE_MB*MB;
+      if (sortBufferByteSize <  minBufferSizeBytes
+          || totalAvailableBytes > 10 * minBufferSizeBytes) { // lets see if we need/should
to grow the heap 
+        if (totalAvailableBytes/2 > minBufferSizeBytes){ // there is enough mem for a
reasonable buffer
+          sortBufferByteSize = totalAvailableBytes/2; // grow the heap
+        } else {
+          //heap seems smallish lets be conservative fall back to the free/2 
+          sortBufferByteSize = Math.max(ABSOLUTE_MIN_SORT_BUFFER_SIZE, sortBufferByteSize);
+        }
       }
-      
-      // by max mem (heap will grow)
-      half = (max - total) / 2;
-      return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+      return new BufferSize(Math.min(Integer.MAX_VALUE, sortBufferByteSize));
     }
   }
-
+  
   /**
    * Sort info (debugging mostly).
    */
@@ -490,4 +496,4 @@ public final class Sort {
   public Comparator<BytesRef> getComparator() {
     return comparator;
   }  
-}
\ No newline at end of file
+}



Mime
View raw message