lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1380808 - in /lucene/dev/trunk: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/store/ lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/ lucene/test-framework/src/java/org/apache/lucene/store/ lucene/test-...
Date Tue, 04 Sep 2012 18:52:20 GMT
Author: mikemccand
Date: Tue Sep  4 18:52:19 2012
New Revision: 1380808

URL: http://svn.apache.org/viewvc?rev=1380808&view=rev
Log:
fix test/test-framework/real bug causing OOME in TestPostingsFormat

Modified:
    lucene/dev/trunk/   (props changed)
    lucene/dev/trunk/lucene/   (props changed)
    lucene/dev/trunk/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/trunk/lucene/core/   (props changed)
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
    lucene/dev/trunk/lucene/test-framework/   (props changed)
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
    lucene/dev/trunk/solr/build.xml   (props changed)

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Sep  4 18:52:19 2012
@@ -138,6 +138,10 @@ Bug Fixes
 * LUCENE-4333: Fixed NPE in TermGroupFacetCollector when faceting on mv fields.
   (Jesse MacVicar, Martijn van Groningen)
 
+* NRTCachingDirectory was always caching a newly flushed segment in
+  RAM, instead of checking the estimated size of the segment
+  to decide whether to cache it. (Mike McCandless)
+
 Optimizations
 
 * LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had increased a

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
(original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
Tue Sep  4 18:52:19 2012
@@ -267,9 +267,16 @@ public class NRTCachingDirectory extends
   /** Subclass can override this to customize logic; return
    *  true if this file should be written to the RAMDirectory. */
   protected boolean doCacheWrite(String name, IOContext context) {
-    final MergeInfo merge = context.mergeInfo;
     //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge
+ " size=" + (merge==null ? 0 : merge.estimatedMergeBytes));
-    return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes
<= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes;
+
+    long bytes = 0;
+    if (context.mergeInfo != null) {
+      bytes = context.mergeInfo.estimatedMergeBytes;
+    } else if (context.flushInfo != null) {
+      bytes = context.flushInfo.estimatedSegmentSize;
+    }
+
+    return !name.equals(IndexFileNames.SEGMENTS_GEN) && (bytes <= maxMergeSizeBytes)
&& (bytes + cache.sizeInBytes()) <= maxCachedBytes;
   }
 
   private final Object uncacheLock = new Object();

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
(original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
Tue Sep  4 18:52:19 2012
@@ -38,12 +38,14 @@ import org.apache.lucene.codecs.TermStat
 import org.apache.lucene.codecs.TermsConsumer;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FlushInfo;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util._TestUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -379,10 +381,13 @@ public class TestPostingsFormat extends 
 
     FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);
 
+    // Estimate that flushed segment size will be 25% of
+    // what we use in RAM:
+    long bytes =  RamUsageEstimator.sizeOf(fields)/4;
+
     SegmentWriteState writeState = new SegmentWriteState(null, dir,
                                                          segmentInfo, newFieldInfos,
-                                                         32, null, IOContext.DEFAULT);
-
+                                                         32, null, new IOContext(new FlushInfo(maxDocID,
bytes)));
     FieldsConsumer fieldsConsumer = Codec.getDefault().postingsFormat().fieldsConsumer(writeState);
 
     for(Map.Entry<String,Map<BytesRef,List<Posting>>> fieldEnt : fields.entrySet())
{

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
Tue Sep  4 18:52:19 2012
@@ -443,7 +443,7 @@ public class MockDirectoryWrapper extend
     }
     
     //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
-    IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)),
name);
+    IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState,
context)), name);
     addFileHandle(io, name, Handle.Output);
     openFilesForWrite.add(name);
     
@@ -497,7 +497,7 @@ public class MockDirectoryWrapper extend
       throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is
still open for writing"), name, false);
     }
 
-    IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState)));
+    IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState,
context)));
     addFileHandle(ii, name, Handle.Input);
     return ii;
   }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1380808&r1=1380807&r2=1380808&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
(original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Tue Sep  4 18:52:19 2012
@@ -1130,10 +1130,25 @@ public abstract class LuceneTestCase ext
 
   /** TODO: javadoc */
   public static IOContext newIOContext(Random random) {
+    return newIOContext(random, IOContext.DEFAULT);
+  }
+
+  /** TODO: javadoc */
+  public static IOContext newIOContext(Random random, IOContext oldContext) {
     final int randomNumDocs = random.nextInt(4192);
     final int size = random.nextInt(512) * randomNumDocs;
-    final IOContext context;
-    switch (random.nextInt(5)) {
+    if (oldContext.flushInfo != null) {
+      // Always return at least the estimatedSegmentSize of
+      // the incoming IOContext:
+      return new IOContext(new FlushInfo(randomNumDocs, Math.max(oldContext.flushInfo.estimatedSegmentSize,
size)));
+    } else if (oldContext.mergeInfo != null) {
+      // Always return at least the estimatedMergeBytes of
+      // the incoming IOContext:
+      return new IOContext(new MergeInfo(randomNumDocs, Math.max(oldContext.mergeInfo.estimatedMergeBytes,
size), random.nextBoolean(), _TestUtil.nextInt(random, 1, 100)));
+    } else {
+      // Make a totally random IOContext:
+      final IOContext context;
+      switch (random.nextInt(5)) {
       case 0:
         context = IOContext.DEFAULT;
         break;
@@ -1151,8 +1166,9 @@ public abstract class LuceneTestCase ext
         break;
       default:
         context = IOContext.DEFAULT;
+      }
+      return context;
     }
-    return context;
   }
 
   /**



Mime
View raw message