lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r926020 - in /lucene/solr/branches/newtrunk/lucene: ./ src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Date Mon, 22 Mar 2010 10:21:08 GMT
Author: mikemccand
Date: Mon Mar 22 10:21:07 2010
New Revision: 926020

URL: http://svn.apache.org/viewvc?rev=926020&view=rev
Log:
LUCENE-2297: allow reader pooling to be enabled (through IndexWriterConfig) even if you're
not pulling NRT readers

Modified:
    lucene/solr/branches/newtrunk/lucene/CHANGES.txt
    lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
    lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
    lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java

Modified: lucene/solr/branches/newtrunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/CHANGES.txt?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/CHANGES.txt (original)
+++ lucene/solr/branches/newtrunk/lucene/CHANGES.txt Mon Mar 22 10:21:07 2010
@@ -193,6 +193,10 @@ New features
   documents (previously this was hardwired to 5), using
   IndexWriterConfig.setMaxThreadStates.  (Mike McCandless)
 
+* LUCENE-2297: Enable turning on reader pooling inside IndexWriter
+  even when getReader (near-real-timer reader) is not in use, through
+  IndexWriterConfig.enable/disableReaderPooling.  (Mike McCandless)
+
 Optimizations
 
 * LUCENE-2075: Terms dict cache is now shared across threads instead

Modified: lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
(original)
+++ lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
Mon Mar 22 10:21:07 2010
@@ -1081,7 +1081,8 @@ public class IndexWriter implements Clos
     mergePolicy.setIndexWriter(this);
     mergeScheduler = conf.getMergeScheduler();
     mergedSegmentWarmer = conf.getMergedSegmentWarmer();
-    
+    poolReaders = conf.getReaderPooling();
+
     OpenMode mode = conf.getOpenMode();
     boolean create;
     if (mode == OpenMode.CREATE) {

Modified: lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
(original)
+++ lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
Mon Mar 22 10:21:07 2010
@@ -84,6 +84,9 @@ public final class IndexWriterConfig imp
    *  others to finish. */
   public final static int DEFAULT_MAX_THREAD_STATES = 8;
 
+  /** Default setting for {@link #setIndexWriterPooling}. */
+  public final static boolean DEFAULT_READER_POOLING = false;
+
   /**
    * Sets the default (for any instance) maximum time to wait for a write lock
    * (in milliseconds).
@@ -118,6 +121,7 @@ public final class IndexWriterConfig imp
   private IndexReaderWarmer mergedSegmentWarmer;
   private MergePolicy mergePolicy;
   private int maxThreadStates;
+  private boolean readerPooling;
   
   // required for clone
   private Version matchVersion;
@@ -147,6 +151,7 @@ public final class IndexWriterConfig imp
     mergedSegmentWarmer = null;
     mergePolicy = new LogByteSizeMergePolicy();
     maxThreadStates = DEFAULT_MAX_THREAD_STATES;
+    readerPooling = DEFAULT_READER_POOLING;
   }
   
   @Override
@@ -506,6 +511,15 @@ public final class IndexWriterConfig imp
   }
   
   /**
+   * Returns the current MergePolicy in use by this writer.
+   * 
+   * @see #setMergePolicy(MergePolicy)
+   */
+  public MergePolicy getMergePolicy() {
+    return mergePolicy;
+  }
+
+  /**
    * Sets the max number of simultaneous threads that may be indexing documents
    * at once in IndexWriter. Values < 1 are invalid and if passed
    * <code>maxThreadStates</code> will be set to
@@ -522,13 +536,23 @@ public final class IndexWriterConfig imp
     return maxThreadStates;
   }
 
-  /**
-   * Returns the current MergePolicy in use by this writer.
-   * 
-   * @see #setMergePolicy(MergePolicy)
-   */
-  public MergePolicy getMergePolicy() {
-    return mergePolicy;
+  /** By default, IndexWriter does not pool the
+   *  SegmentReaders it must open for deletions and
+   *  merging, unless a near-real-time reader has been
+   *  obtained by calling {@link IndexWriter#getReader}.
+   *  This method lets you enable pooling without getting a
+   *  near-real-time reader.  NOTE: if you set this to
+   *  false, IndexWriter will still pool readers once
+   *  {@link IndexWriter#getReader} is called. */
+  public IndexWriterConfig setReaderPooling(boolean readerPooling) {
+    this.readerPooling = readerPooling;
+    return this;
+  }
+
+  /** Returns true if IndexWriter should pool readers even
+   *  if {@link IndexWriter#getReader} has not been called. */
+  public boolean getReaderPooling() {
+    return readerPooling;
   }
 
   /** Expert: sets the {@link DocConsumer} chain to be used to process documents. */
@@ -562,6 +586,7 @@ public final class IndexWriterConfig imp
     sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n");
     sb.append("mergePolicy=").append(mergePolicy).append("\n");
     sb.append("maxThreadStates=").append(maxThreadStates).append("\n");
+    sb.append("readerPooling=").append(readerPooling).append("\n");
     return sb.toString();
   }
 }

Modified: lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
(original)
+++ lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
Mon Mar 22 10:21:07 2010
@@ -78,6 +78,7 @@ public class TestIndexWriterConfig exten
     assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS, conf.getMaxBufferedDeleteTerms());
     assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, conf.getRAMBufferSizeMB(),
0.0);
     assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS, conf.getMaxBufferedDocs());
+    assertEquals(IndexWriterConfig.DEFAULT_READER_POOLING, conf.getReaderPooling());
     assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain());
     assertNull(conf.getMergedSegmentWarmer());
     assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
@@ -102,6 +103,7 @@ public class TestIndexWriterConfig exten
     getters.add("getMergedSegmentWarmer");
     getters.add("getMergePolicy");
     getters.add("getMaxThreadStates");
+    getters.add("getReaderPooling");
     for (Method m : IndexWriterConfig.class.getDeclaredMethods()) {
       if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get"))
{
         assertTrue("method " + m.getName() + " is not tested for defaults", getters.contains(m.getName()));
@@ -133,6 +135,7 @@ public class TestIndexWriterConfig exten
     assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS);
     assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
     assertEquals(16.0, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, 0.0);
+    assertEquals(false, IndexWriterConfig.DEFAULT_READER_POOLING);
     assertEquals(8, IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
   }
   

Modified: lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
(original)
+++ lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
Mon Mar 22 10:21:07 2010
@@ -85,7 +85,8 @@ public class TestStressIndexing2 extends
     Directory dir2 = new MockRAMDirectory();
     // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
     int maxThreadStates = 1+r.nextInt(10);
-    Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates);
+    boolean doReaderPooling = r.nextBoolean();
+    Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates, doReaderPooling);
     indexSerial(docs, dir2);
 
     // verifying verify
@@ -103,6 +104,7 @@ public class TestStressIndexing2 extends
       mergeFactor=r.nextInt(3)+2;
       maxBufferedDocs=r.nextInt(3)+2;
       int maxThreadStates = 1+r.nextInt(10);
+      boolean doReaderPooling = r.nextBoolean();
       seed++;
 
       int nThreads=r.nextInt(5)+1;
@@ -110,7 +112,7 @@ public class TestStressIndexing2 extends
       int range=r.nextInt(20)+1;
       Directory dir1 = new MockRAMDirectory();
       Directory dir2 = new MockRAMDirectory();
-      Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates);
+      Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates,
doReaderPooling);
       indexSerial(docs, dir2);
       verifyEquals(dir1, dir2, "id");
     }
@@ -184,12 +186,14 @@ public class TestStressIndexing2 extends
     return dw;
   }
   
-  public Map<String,Document> indexRandom(int nThreads, int iterations, int range,
Directory dir, int maxThreadStates) throws IOException, InterruptedException {
+  public Map<String,Document> indexRandom(int nThreads, int iterations, int range,
Directory dir, int maxThreadStates,
+                                          boolean doReaderPooling) throws IOException, InterruptedException
{
     Map<String,Document> docs = new HashMap<String,Document>();
     for(int iter=0;iter<3;iter++) {
       IndexWriter w = new MockIndexWriter(dir, new IndexWriterConfig(
           TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)
-               .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates));
+               .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates)
+               .setReaderPooling(doReaderPooling));
       LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
       lmp.setUseCompoundFile(false);
       lmp.setUseCompoundDocStore(false);



Mime
View raw message