lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1441483 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/core/ core/src/java/org/apache/solr/update/ core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/update/
Date Fri, 01 Feb 2013 15:29:09 GMT
Author: markrmiller
Date: Fri Feb  1 15:29:08 2013
New Revision: 1441483

URL: http://svn.apache.org/viewvc?rev=1441483&view=rev
Log:
SOLR-4370: Allow configuring commitWithin to do hard commits.

Added:
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java   (with
props)
Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb  1 15:29:08 2013
@@ -68,6 +68,9 @@ New Features
 
 * SOLR-2827: RegexpBoost Update Processor (janhoy)
 
+* SOLR-4370: Allow configuring commitWithin to do hard commits. 
+  (Mark Miller, Senthuran Sivananthan)
+
 Bug Fixes
 ----------------------
 
@@ -97,11 +100,6 @@ Bug Fixes
 
 * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
 
-* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
-
-* SOLR-4380: Replicate after startup option would not replicate until the
-  IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
-
 Optimizations
 ----------------------
 
@@ -119,11 +117,6 @@ Optimizations
 * SOLR-4306: Utilize indexInfo=false when gathering core names in UI
   (steffkes)
 
-* SOLR-4284: Admin UI - make core list scrollable separate from the rest of
-  the UI (steffkes)
-
-* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
-
 Other Changes
 ----------------------
 
@@ -132,8 +125,6 @@ Other Changes
 
 * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
 
-* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
-
 ==================  4.1.0 ==================
 
 Versions of Major Components

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java Fri Feb  1 15:29:08
2013
@@ -238,7 +238,8 @@ public class SolrConfig extends Config {
             getBool("updateHandler/autoCommit/openSearcher",true),
             getInt("updateHandler/commitIntervalLowerBound",-1),
             getInt("updateHandler/autoSoftCommit/maxDocs",-1),
-            getInt("updateHandler/autoSoftCommit/maxTime",-1));
+            getInt("updateHandler/autoSoftCommit/maxTime",-1),
+            getBool("updateHandler/commitWithin/softCommit",true));
   }
 
   private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass)
{
@@ -402,6 +403,7 @@ public class SolrConfig extends Config {
     public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
         autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
     public final boolean openSearcher;  // is opening a new searcher part of hard autocommit?
+    public final boolean commitWithinSoftCommit;
 
     /**
      * @param autoCommmitMaxDocs set -1 as default
@@ -409,7 +411,7 @@ public class SolrConfig extends Config {
      * @param commitIntervalLowerBound set -1 as default
      */
     public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime,
boolean openSearcher, int commitIntervalLowerBound,
-        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
+        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit)
{
       this.className = className;
       this.autoCommmitMaxDocs = autoCommmitMaxDocs;
       this.autoCommmitMaxTime = autoCommmitMaxTime;
@@ -418,6 +420,8 @@ public class SolrConfig extends Config {
       
       this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
       this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
+      
+      this.commitWithinSoftCommit = commitWithinSoftCommit;
     } 
   }
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Fri
Feb  1 15:29:08 2013
@@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend
   // tracks when auto-commit should occur
   protected final CommitTracker commitTracker;
   protected final CommitTracker softCommitTracker;
+  
+  protected boolean commitWithinSoftCommit;
 
   public DirectUpdateHandler2(SolrCore core) {
     super(core);
@@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend
     int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs",
-1);
     int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime",
-1);
     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound,
true, true);
+    
+    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
   }
   
   public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
@@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend
     if (this.ulog != null) {
       this.ulog.init(this, core);
     }
+    
+    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
   }
 
   private void deleteAll() throws IOException {
@@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend
         }
         
         if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
-          commitTracker.addedDocument(-1);
-          softCommitTracker.addedDocument(cmd.commitWithin);
+          if (commitWithinSoftCommit) {
+            commitTracker.addedDocument(-1);
+            softCommitTracker.addedDocument(cmd.commitWithin);
+          } else {
+            softCommitTracker.addedDocument(-1);
+            commitTracker.addedDocument(cmd.commitWithin);
+          }
         }
         
         rc = 1;
@@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend
   
   private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
     if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
-      softCommitTracker.deletedDocument(cmd.commitWithin);
+      if (commitWithinSoftCommit) {
+        softCommitTracker.deletedDocument(cmd.commitWithin);
+      } else {
+        commitTracker.deletedDocument(cmd.commitWithin);
+      }
       
       if (commitTracker.getTimeUpperBound() > 0) {
         commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());

Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Fri Feb
 1 15:29:08 2013
@@ -97,6 +97,10 @@
     <updateLog enable="${enable.update.log:false}">
   	  <str name="dir">${solr.ulog.dir:}</str>
     </updateLog> 
+    
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
 
   </updateHandler>
 

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java Fri Feb
 1 15:29:08 2013
@@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst
     
     MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
     
-    // Add a single document with commitWithin == 1 second
+    // Add a single document with commitWithin == 2 second
     SolrQueryResponse rsp = new SolrQueryResponse();
     SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
     req.setContentStreams( toContentStreams(

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java Fri
Feb  1 15:29:08 2013
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+
+import java.util.HashMap;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.UpdateRequestHandler;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+@Slow
+public class HardAutoCommitTest extends AbstractSolrTestCase {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    System.setProperty("solr.commitwithin.softcommit", "false");
+    initCore("solrconfig.xml", "schema.xml");
+  }
+  
+  @AfterClass
+  public static void afterClass() {
+    System.clearProperty("solr.commitwithin.softcommit");
+  }
+
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    clearIndex();
+    // reload the core to clear stats
+    h.getCoreContainer().reload(h.getCore().getName());
+  }
+
+
+  public void testCommitWithin() throws Exception {
+    SolrCore core = h.getCore();
+    
+    NewSearcherListener trigger = new NewSearcherListener();    
+    core.registerNewSearcherListener(trigger);
+    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
+    CommitTracker tracker = updater.commitTracker;
+    tracker.setTimeUpperBound(0);
+    tracker.setDocsUpperBound(-1);
+    
+    UpdateRequestHandler handler = new UpdateRequestHandler();
+    handler.init( null );
+    
+    MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
+    
+    // Add a single document with commitWithin == 2 second
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+      adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null ) );
+    trigger.reset();
+    handler.handleRequest( req, rsp );
+
+    // Check it isn't in the index
+    assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
+    
+    // Wait longer than the commitWithin time
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+
+    // Add one document without commitWithin
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
+      trigger.reset();
+      handler.handleRequest( req, rsp );
+      
+    // Check it isn't in the index
+    assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" );
+    
+    // Delete one document with commitWithin
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+      delI("529", "commitWithin", "1000"), null ) );
+    trigger.reset();
+    handler.handleRequest( req, rsp );
+      
+    // Now make sure we can find it
+    assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
+    
+    // Wait for the commit to happen
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+    
+    // Now we shouldn't find it
+    assertQ("should find none", req("id:529") ,"//result[@numFound=0]" );
+    // ... but we should find the new one
+    assertQ("should find one", req("id:530") ,"//result[@numFound=1]" );
+    
+    trigger.reset();
+    
+    // now make the call 10 times really fast and make sure it 
+    // only commits once
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        adoc(2000, "id", "500" ), null ) );
+    for( int i=0;i<10; i++ ) {
+      handler.handleRequest( req, rsp );
+    }
+    assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
+    
+    // the same for the delete
+    req.setContentStreams( AutoCommitTest.toContentStreams(
+        delI("530", "commitWithin", "1000"), null ) );
+    for( int i=0;i<10; i++ ) {
+      handler.handleRequest( req, rsp );
+    }
+    assertQ("should be there", req("id:530") ,"//result[@numFound=1]" );
+    
+    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
+    assertQ("should be there", req("id:500") ,"//result[@numFound=1]" );
+    assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" );
+    
+    assertEquals(3, tracker.getCommitCount());
+  }
+
+}



Mime
View raw message