lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Mark Miller <markrmil...@gmail.com>
Subject Re: svn commit: r1441483 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/core/ core/src/java/org/apache/solr/update/ core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/update/
Date Fri, 01 Feb 2013 16:40:10 GMT
woah, def not - bad merge of somehow, thanks.

- Mark

On Feb 1, 2013, at 11:12 AM, Alan Woodward <alan@flax.co.uk> wrote:

> Hey Mark, did you mean to back out these CHANGES.TXT entries?
> 
> Alan Woodward
> www.flax.co.uk
> 
> 
> On 1 Feb 2013, at 15:29, markrmiller@apache.org wrote:
> 
>> Author: markrmiller
>> Date: Fri Feb  1 15:29:08 2013
>> New Revision: 1441483
>> 
>> URL: http://svn.apache.org/viewvc?rev=1441483&view=rev
>> Log:
>> SOLR-4370: Allow configuring commitWithin to do hard commits.
>> 
>> Added:
>>    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
  (with props)
>> Modified:
>>    lucene/dev/trunk/solr/CHANGES.txt
>>    lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
>>    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>>    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
>>    lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
>> 
>> Modified: lucene/dev/trunk/solr/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/CHANGES.txt (original)
>> +++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb  1 15:29:08 2013
>> @@ -68,6 +68,9 @@ New Features
>> 
>> * SOLR-2827: RegexpBoost Update Processor (janhoy)
>> 
>> +* SOLR-4370: Allow configuring commitWithin to do hard commits. 
>> +  (Mark Miller, Senthuran Sivananthan)
>> +
>> Bug Fixes
>> ----------------------
>> 
>> @@ -97,11 +100,6 @@ Bug Fixes
>> 
>> * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
>> 
>> -* SOLR-3967: langid.enforceSchema option checks source field instead of target field
(janhoy)
>> -
>> -* SOLR-4380: Replicate after startup option would not replicate until the
>> -  IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
>> -
>> Optimizations
>> ----------------------
>> 
>> @@ -119,11 +117,6 @@ Optimizations
>> * SOLR-4306: Utilize indexInfo=false when gathering core names in UI
>>   (steffkes)
>> 
>> -* SOLR-4284: Admin UI - make core list scrollable separate from the rest of
>> -  the UI (steffkes)
>> -
>> -* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
>> -
>> Other Changes
>> ----------------------
>> 
>> @@ -132,8 +125,6 @@ Other Changes
>> 
>> * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
>> 
>> -* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
>> -
>> ==================  4.1.0 ==================
>> 
>> Versions of Major Components
>> 
>> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
>> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java Fri
Feb  1 15:29:08 2013
>> @@ -238,7 +238,8 @@ public class SolrConfig extends Config {
>>             getBool("updateHandler/autoCommit/openSearcher",true),
>>             getInt("updateHandler/commitIntervalLowerBound",-1),
>>             getInt("updateHandler/autoSoftCommit/maxDocs",-1),
>> -            getInt("updateHandler/autoSoftCommit/maxTime",-1));
>> +            getInt("updateHandler/autoSoftCommit/maxTime",-1),
>> +            getBool("updateHandler/commitWithin/softCommit",true));
>>   }
>> 
>>   private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean
requireClass) {
>> @@ -402,6 +403,7 @@ public class SolrConfig extends Config {
>>     public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound,
>>         autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime;
>>     public final boolean openSearcher;  // is opening a new searcher part of hard
autocommit?
>> +    public final boolean commitWithinSoftCommit;
>> 
>>     /**
>>      * @param autoCommmitMaxDocs set -1 as default
>> @@ -409,7 +411,7 @@ public class SolrConfig extends Config {
>>      * @param commitIntervalLowerBound set -1 as default
>>      */
>>     public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime,
boolean openSearcher, int commitIntervalLowerBound,
>> -        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) {
>> +        int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit)
{
>>       this.className = className;
>>       this.autoCommmitMaxDocs = autoCommmitMaxDocs;
>>       this.autoCommmitMaxTime = autoCommmitMaxTime;
>> @@ -418,6 +420,8 @@ public class SolrConfig extends Config {
>> 
>>       this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs;
>>       this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime;
>> +      
>> +      this.commitWithinSoftCommit = commitWithinSoftCommit;
>>     } 
>>   }
>> 
>> 
>> Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
(original)
>> +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
Fri Feb  1 15:29:08 2013
>> @@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend
>>   // tracks when auto-commit should occur
>>   protected final CommitTracker commitTracker;
>>   protected final CommitTracker softCommitTracker;
>> +  
>> +  protected boolean commitWithinSoftCommit;
>> 
>>   public DirectUpdateHandler2(SolrCore core) {
>>     super(core);
>> @@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend
>>     int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs",
-1);
>>     int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime",
-1);
>>     softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound,
softCommitTimeUpperBound, true, true);
>> +    
>> +    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
>>   }
>> 
>>   public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) {
>> @@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend
>>     if (this.ulog != null) {
>>       this.ulog.init(this, core);
>>     }
>> +    
>> +    commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
>>   }
>> 
>>   private void deleteAll() throws IOException {
>> @@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend
>>         }
>> 
>>         if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
>> -          commitTracker.addedDocument(-1);
>> -          softCommitTracker.addedDocument(cmd.commitWithin);
>> +          if (commitWithinSoftCommit) {
>> +            commitTracker.addedDocument(-1);
>> +            softCommitTracker.addedDocument(cmd.commitWithin);
>> +          } else {
>> +            softCommitTracker.addedDocument(-1);
>> +            commitTracker.addedDocument(cmd.commitWithin);
>> +          }
>>         }
>> 
>>         rc = 1;
>> @@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend
>> 
>>   private void updateDeleteTrackers(DeleteUpdateCommand cmd) {
>>     if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) {
>> -      softCommitTracker.deletedDocument(cmd.commitWithin);
>> +      if (commitWithinSoftCommit) {
>> +        softCommitTracker.deletedDocument(cmd.commitWithin);
>> +      } else {
>> +        commitTracker.deletedDocument(cmd.commitWithin);
>> +      }
>> 
>>       if (commitTracker.getTimeUpperBound() > 0) {
>>         commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound());
>> 
>> Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
(original)
>> +++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
Fri Feb  1 15:29:08 2013
>> @@ -97,6 +97,10 @@
>>     <updateLog enable="${enable.update.log:false}">
>>   	  <str name="dir">${solr.ulog.dir:}</str>
>>     </updateLog> 
>> +    
>> +    <commitWithin>
>> +      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
>> +    </commitWithin>
>> 
>>   </updateHandler>
>> 
>> 
>> Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
(original)
>> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java
Fri Feb  1 15:29:08 2013
>> @@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst
>> 
>>     MapSolrParams params = new MapSolrParams( new HashMap<String, String>()
);
>> 
>> -    // Add a single document with commitWithin == 1 second
>> +    // Add a single document with commitWithin == 2 second
>>     SolrQueryResponse rsp = new SolrQueryResponse();
>>     SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
>>     req.setContentStreams( toContentStreams(
>> 
>> Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto
>> ==============================================================================
>> --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
(added)
>> +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java
Fri Feb  1 15:29:08 2013
>> @@ -0,0 +1,136 @@
>> +/*
>> + * Licensed to the Apache Software Foundation (ASF) under one or more
>> + * contributor license agreements.  See the NOTICE file distributed with
>> + * this work for additional information regarding copyright ownership.
>> + * The ASF licenses this file to You under the Apache License, Version 2.0
>> + * (the "License"); you may not use this file except in compliance with
>> + * the License.  You may obtain a copy of the License at
>> + *
>> + *     http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +
>> +package org.apache.solr.update;
>> +
>> +import java.util.HashMap;
>> +
>> +import org.apache.lucene.util.LuceneTestCase.Slow;
>> +import org.apache.solr.common.params.MapSolrParams;
>> +import org.apache.solr.core.SolrCore;
>> +import org.apache.solr.handler.UpdateRequestHandler;
>> +import org.apache.solr.request.SolrQueryRequestBase;
>> +import org.apache.solr.response.SolrQueryResponse;
>> +import org.apache.solr.util.AbstractSolrTestCase;
>> +import org.junit.AfterClass;
>> +import org.junit.BeforeClass;
>> +
>> +@Slow
>> +public class HardAutoCommitTest extends AbstractSolrTestCase {
>> +
>> +  @BeforeClass
>> +  public static void beforeClass() throws Exception {
>> +    System.setProperty("solr.commitwithin.softcommit", "false");
>> +    initCore("solrconfig.xml", "schema.xml");
>> +  }
>> +  
>> +  @AfterClass
>> +  public static void afterClass() {
>> +    System.clearProperty("solr.commitwithin.softcommit");
>> +  }
>> +
>> +  @Override
>> +  public void setUp() throws Exception {
>> +    super.setUp();
>> +    clearIndex();
>> +    // reload the core to clear stats
>> +    h.getCoreContainer().reload(h.getCore().getName());
>> +  }
>> +
>> +
>> +  public void testCommitWithin() throws Exception {
>> +    SolrCore core = h.getCore();
>> +    
>> +    NewSearcherListener trigger = new NewSearcherListener();    
>> +    core.registerNewSearcherListener(trigger);
>> +    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
>> +    CommitTracker tracker = updater.commitTracker;
>> +    tracker.setTimeUpperBound(0);
>> +    tracker.setDocsUpperBound(-1);
>> +    
>> +    UpdateRequestHandler handler = new UpdateRequestHandler();
>> +    handler.init( null );
>> +    
>> +    MapSolrParams params = new MapSolrParams( new HashMap<String, String>()
);
>> +    
>> +    // Add a single document with commitWithin == 2 second
>> +    SolrQueryResponse rsp = new SolrQueryResponse();
>> +    SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {};
>> +    req.setContentStreams( AutoCommitTest.toContentStreams(
>> +      adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null
) );
>> +    trigger.reset();
>> +    handler.handleRequest( req, rsp );
>> +
>> +    // Check it isn't in the index
>> +    assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" );
>> +    
>> +    // Wait longer than the commitWithin time
>> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
>> +
>> +    // Add one document without commitWithin
>> +    req.setContentStreams( AutoCommitTest.toContentStreams(
>> +        adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null
) );
>> +      trigger.reset();
>> +      handler.handleRequest( req, rsp );
>> +      
>> +    // Check it isn't in the index
>> +    assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" );
>> +    
>> +    // Delete one document with commitWithin
>> +    req.setContentStreams( AutoCommitTest.toContentStreams(
>> +      delI("529", "commitWithin", "1000"), null ) );
>> +    trigger.reset();
>> +    handler.handleRequest( req, rsp );
>> +      
>> +    // Now make sure we can find it
>> +    assertQ("should find one", req("id:529") ,"//result[@numFound=1]" );
>> +    
>> +    // Wait for the commit to happen
>> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
>> +    
>> +    // Now we shouldn't find it
>> +    assertQ("should find none", req("id:529") ,"//result[@numFound=0]" );
>> +    // ... but we should find the new one
>> +    assertQ("should find one", req("id:530") ,"//result[@numFound=1]" );
>> +    
>> +    trigger.reset();
>> +    
>> +    // now make the call 10 times really fast and make sure it 
>> +    // only commits once
>> +    req.setContentStreams( AutoCommitTest.toContentStreams(
>> +        adoc(2000, "id", "500" ), null ) );
>> +    for( int i=0;i<10; i++ ) {
>> +      handler.handleRequest( req, rsp );
>> +    }
>> +    assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" );
>> +    
>> +    // the same for the delete
>> +    req.setContentStreams( AutoCommitTest.toContentStreams(
>> +        delI("530", "commitWithin", "1000"), null ) );
>> +    for( int i=0;i<10; i++ ) {
>> +      handler.handleRequest( req, rsp );
>> +    }
>> +    assertQ("should be there", req("id:530") ,"//result[@numFound=1]" );
>> +    
>> +    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
>> +    assertQ("should be there", req("id:500") ,"//result[@numFound=1]" );
>> +    assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" );
>> +    
>> +    assertEquals(3, tracker.getCommitCount());
>> +  }
>> +
>> +}
>> 
>> 
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Mime
View raw message