lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Alan Woodward <a...@flax.co.uk>
Subject Re: svn commit: r1441483 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/core/ core/src/java/org/apache/solr/update/ core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/update/
Date Fri, 01 Feb 2013 16:12:05 GMT
Hey Mark, did you mean to back out these CHANGES.TXT entries?

Alan Woodward
www.flax.co.uk


On 1 Feb 2013, at 15:29, markrmiller@apache.org wrote:

> Author: markrmiller
> Date: Fri Feb  1 15:29:08 2013
> New Revision: 1441483
> 
> URL: http://svn.apache.org/viewvc?rev=1441483&view=rev
> Log:
> SOLR-4370: Allow configuring commitWithin to do hard commits.
> 
> Added:
>    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
  (with props)
> Modified:
>    lucene/dev/trunk/solr/CHANGES.txt
>    lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
>    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
>    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
> 
> Modified: lucene/dev/trunk/solr/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/CHANGES.txt (original)
> +++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb  1 15:29:08 2013
> @@ -68,6 +68,9 @@ New Features
> 
> * SOLR-2827: RegexpBoost Update Processor (janhoy)
> 
> +* SOLR-4370: Allow configuring commitWithin to do hard commits. 
> +  (Mark Miller, Senthuran Sivananthan)
> +
> Bug Fixes
> ----------------------
> 
> @@ -97,11 +100,6 @@ Bug Fixes
> 
> * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
> 
> -* SOLR-3967: langid.enforceSchema option checks source field instead of target field
(janhoy)
> -
> -* SOLR-4380: Replicate after startup option would not replicate until the
> -  IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
> -
> Optimizations
> ----------------------
> 
> @@ -119,11 +117,6 @@ Optimizations
> * SOLR-4306: Utilize indexInfo=false when gathering core names in UI
>   (steffkes)
> 
> -* SOLR-4284: Admin UI - make core list scrollable separate from the rest of
> -  the UI (steffkes)
> -
> -* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
> -
> Other Changes
> ----------------------
> 
> @@ -132,8 +125,6 @@ Other Changes
> 
> * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
> 
> -* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
> -
> ==================  4.1.0 ==================
> 
> Versions of Major Components
> 
> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java Fri Feb
 1 15:29:08 2013
> @@ -238,7 +238,8 @@ public class SolrConfig extends Config {
>             getBool("updateHandler/autoCommit/openSearcher",true),
>             getInt("updateHandler/commitIntervalLowerBound",-1),
>             getInt("updateHandler/autoSoftCommit/maxDocs",-1),
> -            getInt("updateHandler/autoSoftCommit/maxTime",-1));
> +            getInt("updateHandler/autoSoftCommit/maxTime",-1),
> +            getBool("updateHandler/commitWithin/softCommit",true));
>   }
> 
>   private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass)
{
> @@ -402,6 +403,7 @@ public class SolrConfig extends Config {
>     public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
>         autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
>     public final boolean openSearcher;  // is opening a new searcher part of hard autocommit?
> +    public final boolean commitWithinSoftCommit;
> 
>     /**
>      * @param autoCommmitMaxDocs set -1 as default
> @@ -409,7 +411,7 @@ public class SolrConfig extends Config {
>      * @param commitIntervalLowerBound set -1 as default
>      */
>     public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime,
boolean openSearcher, int commitIntervalLowerBound,
> -        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
> +        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit)
{
>       this.className = className;
>       this.autoCommmitMaxDocs = autoCommmitMaxDocs;
>       this.autoCommmitMaxTime = autoCommmitMaxTime;
> @@ -418,6 +420,8 @@ public class SolrConfig extends Config {
> 
>       this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
>       this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
> +      
> +      this.commitWithinSoftCommit = commitWithinSoftCommit;
>     } 
>   }
> 
> 
> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
(original)
> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
Fri Feb  1 15:29:08 2013
> @@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend
>   // tracks when auto-commit should occur
>   protected final CommitTracker commitTracker;
>   protected final CommitTracker softCommitTracker;
> +  
> +  protected boolean commitWithinSoftCommit;
> 
>   public DirectUpdateHandler2(SolrCore core) {
>     super(core);
> @@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend
>     int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs",
-1);
>     int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime",
-1);
>     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound,
true, true);
> +    
> +    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
>   }
> 
>   public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
> @@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend
>     if (this.ulog != null) {
>       this.ulog.init(this, core);
>     }
> +    
> +    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
>   }
> 
>   private void deleteAll() throws IOException {
> @@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend
>         }
> 
>         if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
> -          commitTracker.addedDocument(-1);
> -          softCommitTracker.addedDocument(cmd.commitWithin);
> +          if (commitWithinSoftCommit) {
> +            commitTracker.addedDocument(-1);
> +            softCommitTracker.addedDocument(cmd.commitWithin);
> +          } else {
> +            softCommitTracker.addedDocument(-1);
> +            commitTracker.addedDocument(cmd.commitWithin);
> +          }
>         }
> 
>         rc = 1;
> @@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend
> 
>   private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
>     if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
> -      softCommitTracker.deletedDocument(cmd.commitWithin);
> +      if (commitWithinSoftCommit) {
> +        softCommitTracker.deletedDocument(cmd.commitWithin);
> +      } else {
> +        commitTracker.deletedDocument(cmd.commitWithin);
> +      }
> 
>       if (commitTracker.getTimeUpperBound() > 0) {
>         commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
> 
> Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original)
> +++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Fri
Feb  1 15:29:08 2013
> @@ -97,6 +97,10 @@
>     <updateLog enable="${enable.update.log:false}">
>   	  <str name="dir">${solr.ulog.dir:}</str>
>     </updateLog> 
> +    
> +    <commitWithin>
> +      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
> +    </commitWithin>
> 
>   </updateHandler>
> 
> 
> Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java (original)
> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java Fri
Feb  1 15:29:08 2013
> @@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst
> 
>     MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
> 
> -    // Add a single document with commitWithin == 1 second
> +    // Add a single document with commitWithin == 2 second
>     SolrQueryResponse rsp = new SolrQueryResponse();
>     SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
>     req.setContentStreams( toContentStreams(
> 
> Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto
> ==============================================================================
> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
(added)
> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
Fri Feb  1 15:29:08 2013
> @@ -0,0 +1,136 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *     http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +package org.apache.solr.update;
> +
> +import java.util.HashMap;
> +
> +import org.apache.lucene.util.LuceneTestCase.Slow;
> +import org.apache.solr.common.params.MapSolrParams;
> +import org.apache.solr.core.SolrCore;
> +import org.apache.solr.handler.UpdateRequestHandler;
> +import org.apache.solr.request.SolrQueryRequestBase;
> +import org.apache.solr.response.SolrQueryResponse;
> +import org.apache.solr.util.AbstractSolrTestCase;
> +import org.junit.AfterClass;
> +import org.junit.BeforeClass;
> +
> +@Slow
> +public class HardAutoCommitTest extends AbstractSolrTestCase {
> +
> +  @BeforeClass
> +  public static void beforeClass() throws Exception {
> +    System.setProperty("solr.commitwithin.softcommit", "false");
> +    initCore("solrconfig.xml", "schema.xml");
> +  }
> +  
> +  @AfterClass
> +  public static void afterClass() {
> +    System.clearProperty("solr.commitwithin.softcommit");
> +  }
> +
> +  @Override
> +  public void setUp() throws Exception {
> +    super.setUp();
> +    clearIndex();
> +    // reload the core to clear stats
> +    h.getCoreContainer().reload(h.getCore().getName());
> +  }
> +
> +
> +  public void testCommitWithin() throws Exception {
> +    SolrCore core = h.getCore();
> +    
> +    NewSearcherListener trigger = new NewSearcherListener();    
> +    core.registerNewSearcherListener(trigger);
> +    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
> +    CommitTracker tracker = updater.commitTracker;
> +    tracker.setTimeUpperBound(0);
> +    tracker.setDocsUpperBound(-1);
> +    
> +    UpdateRequestHandler handler = new UpdateRequestHandler();
> +    handler.init( null );
> +    
> +    MapSolrParams params = new MapSolrParams( new HashMap<String, String>() );
> +    
> +    // Add a single document with commitWithin == 2 second
> +    SolrQueryResponse rsp = new SolrQueryResponse();
> +    SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
> +    req.setContentStreams( AutoCommitTest.toContentStreams(
> +      adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null
) );
> +    trigger.reset();
> +    handler.handleRequest( req, rsp );
> +
> +    // Check it isn't in the index
> +    assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
> +    
> +    // Wait longer than the commitWithin time
> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
> +
> +    // Add one document without commitWithin
> +    req.setContentStreams( AutoCommitTest.toContentStreams(
> +        adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) );
> +      trigger.reset();
> +      handler.handleRequest( req, rsp );
> +      
> +    // Check it isn't in the index
> +    assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" );
> +    
> +    // Delete one document with commitWithin
> +    req.setContentStreams( AutoCommitTest.toContentStreams(
> +      delI("529", "commitWithin", "1000"), null ) );
> +    trigger.reset();
> +    handler.handleRequest( req, rsp );
> +      
> +    // Now make sure we can find it
> +    assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
> +    
> +    // Wait for the commit to happen
> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
> +    
> +    // Now we shouldn't find it
> +    assertQ("should find none", req("id:529") ,"//result[@numFound=0]" );
> +    // ... but we should find the new one
> +    assertQ("should find one", req("id:530") ,"//result[@numFound=1]" );
> +    
> +    trigger.reset();
> +    
> +    // now make the call 10 times really fast and make sure it 
> +    // only commits once
> +    req.setContentStreams( AutoCommitTest.toContentStreams(
> +        adoc(2000, "id", "500" ), null ) );
> +    for( int i=0;i<10; i++ ) {
> +      handler.handleRequest( req, rsp );
> +    }
> +    assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
> +    
> +    // the same for the delete
> +    req.setContentStreams( AutoCommitTest.toContentStreams(
> +        delI("530", "commitWithin", "1000"), null ) );
> +    for( int i=0;i<10; i++ ) {
> +      handler.handleRequest( req, rsp );
> +    }
> +    assertQ("should be there", req("id:530") ,"//result[@numFound=1]" );
> +    
> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
> +    assertQ("should be there", req("id:500") ,"//result[@numFound=1]" );
> +    assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" );
> +    
> +    assertEquals(3, tracker.getCommitCount());
> +  }
> +
> +}
> 
> 


Mime
View raw message