Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BD820EF3F for ; Fri, 1 Feb 2013 15:29:31 +0000 (UTC) Received: (qmail 25421 invoked by uid 500); 1 Feb 2013 15:29:30 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 25030 invoked by uid 99); 1 Feb 2013 15:29:30 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Feb 2013 15:29:30 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Feb 2013 15:29:28 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9E31E2388900; Fri, 1 Feb 2013 15:29:09 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1441483 - in /lucene/dev/trunk/solr: ./ core/src/java/org/apache/solr/core/ core/src/java/org/apache/solr/update/ core/src/test-files/solr/collection1/conf/ core/src/test/org/apache/solr/update/ Date: Fri, 01 Feb 2013 15:29:09 -0000 To: commits@lucene.apache.org From: markrmiller@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130201152909.9E31E2388900@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: markrmiller Date: Fri Feb 1 15:29:08 2013 New Revision: 1441483 URL: http://svn.apache.org/viewvc?rev=1441483&view=rev Log: SOLR-4370: Allow configuring commitWithin to do hard commits. Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java (with props) Modified: lucene/dev/trunk/solr/CHANGES.txt lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java Modified: lucene/dev/trunk/solr/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1441483&r1=1441482&r2=1441483&view=diff ============================================================================== --- lucene/dev/trunk/solr/CHANGES.txt (original) +++ lucene/dev/trunk/solr/CHANGES.txt Fri Feb 1 15:29:08 2013 @@ -68,6 +68,9 @@ New Features * SOLR-2827: RegexpBoost Update Processor (janhoy) +* SOLR-4370: Allow configuring commitWithin to do hard commits. + (Mark Miller, Senthuran Sivananthan) + Bug Fixes ---------------------- @@ -97,11 +100,6 @@ Bug Fixes * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman) -* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy) - -* SOLR-4380: Replicate after startup option would not replicate until the - IndexWriter was lazily opened. (Mark Miller, Gregg Donovan) - Optimizations ---------------------- @@ -119,11 +117,6 @@ Optimizations * SOLR-4306: Utilize indexInfo=false when gathering core names in UI (steffkes) -* SOLR-4284: Admin UI - make core list scrollable separate from the rest of - the UI (steffkes) - -* SOLR-4364: Admin UI - Locale based number formatting (steffkes) - Other Changes ---------------------- @@ -132,8 +125,6 @@ Other Changes * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman) -* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy) - ================== 4.1.0 ================== Versions of Major Components Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1441483&r1=1441482&r2=1441483&view=diff ============================================================================== --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original) +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java Fri Feb 1 15:29:08 2013 @@ -238,7 +238,8 @@ public class SolrConfig extends Config { getBool("updateHandler/autoCommit/openSearcher",true), getInt("updateHandler/commitIntervalLowerBound",-1), getInt("updateHandler/autoSoftCommit/maxDocs",-1), - getInt("updateHandler/autoSoftCommit/maxTime",-1)); + getInt("updateHandler/autoSoftCommit/maxTime",-1), + getBool("updateHandler/commitWithin/softCommit",true)); } private void loadPluginInfo(Class clazz, String tag, boolean requireName, boolean requireClass) { @@ -402,6 +403,7 @@ public class SolrConfig extends Config { public final int autoCommmitMaxDocs,autoCommmitMaxTime,commitIntervalLowerBound, autoSoftCommmitMaxDocs,autoSoftCommmitMaxTime; public final boolean openSearcher; // is opening a new searcher part of hard autocommit? + public final boolean commitWithinSoftCommit; /** * @param autoCommmitMaxDocs set -1 as default @@ -409,7 +411,7 @@ public class SolrConfig extends Config { * @param commitIntervalLowerBound set -1 as default */ public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean openSearcher, int commitIntervalLowerBound, - int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime) { + int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) { this.className = className; this.autoCommmitMaxDocs = autoCommmitMaxDocs; this.autoCommmitMaxTime = autoCommmitMaxTime; @@ -418,6 +420,8 @@ public class SolrConfig extends Config { this.autoSoftCommmitMaxDocs = autoSoftCommmitMaxDocs; this.autoSoftCommmitMaxTime = autoSoftCommmitMaxTime; + + this.commitWithinSoftCommit = commitWithinSoftCommit; } } Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1441483&r1=1441482&r2=1441483&view=diff ============================================================================== --- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original) +++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Fri Feb 1 15:29:08 2013 @@ -91,6 +91,8 @@ public class DirectUpdateHandler2 extend // tracks when auto-commit should occur protected final CommitTracker commitTracker; protected final CommitTracker softCommitTracker; + + protected boolean commitWithinSoftCommit; public DirectUpdateHandler2(SolrCore core) { super(core); @@ -106,6 +108,8 @@ public class DirectUpdateHandler2 extend int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1); int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1); softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); + + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; } public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) { @@ -126,6 +130,8 @@ public class DirectUpdateHandler2 extend if (this.ulog != null) { this.ulog.init(this, core); } + + commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; } private void deleteAll() throws IOException { @@ -229,8 +235,13 @@ public class DirectUpdateHandler2 extend } if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { - commitTracker.addedDocument(-1); - softCommitTracker.addedDocument(cmd.commitWithin); + if (commitWithinSoftCommit) { + commitTracker.addedDocument(-1); + softCommitTracker.addedDocument(cmd.commitWithin); + } else { + softCommitTracker.addedDocument(-1); + commitTracker.addedDocument(cmd.commitWithin); + } } rc = 1; @@ -252,7 +263,11 @@ public class DirectUpdateHandler2 extend private void updateDeleteTrackers(DeleteUpdateCommand cmd) { if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { - softCommitTracker.deletedDocument(cmd.commitWithin); + if (commitWithinSoftCommit) { + softCommitTracker.deletedDocument(cmd.commitWithin); + } else { + commitTracker.deletedDocument(cmd.commitWithin); + } if (commitTracker.getTimeUpperBound() > 0) { commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound()); Modified: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1441483&r1=1441482&r2=1441483&view=diff ============================================================================== --- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml (original) +++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml Fri Feb 1 15:29:08 2013 @@ -97,6 +97,10 @@ ${solr.ulog.dir:} + + + ${solr.commitwithin.softcommit:true} + Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java?rev=1441483&r1=1441482&r2=1441483&view=diff ============================================================================== --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java (original) +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/AutoCommitTest.java Fri Feb 1 15:29:08 2013 @@ -286,7 +286,7 @@ public class AutoCommitTest extends Abst MapSolrParams params = new MapSolrParams( new HashMap() ); - // Add a single document with commitWithin == 1 second + // Add a single document with commitWithin == 2 second SolrQueryResponse rsp = new SolrQueryResponse(); SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; req.setContentStreams( toContentStreams( Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java?rev=1441483&view=auto ============================================================================== --- lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java (added) +++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/HardAutoCommitTest.java Fri Feb 1 15:29:08 2013 @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.util.HashMap; + +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.AbstractSolrTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +@Slow +public class HardAutoCommitTest extends AbstractSolrTestCase { + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("solr.commitwithin.softcommit", "false"); + initCore("solrconfig.xml", "schema.xml"); + } + + @AfterClass + public static void afterClass() { + System.clearProperty("solr.commitwithin.softcommit"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + // reload the core to clear stats + h.getCoreContainer().reload(h.getCore().getName()); + } + + + public void testCommitWithin() throws Exception { + SolrCore core = h.getCore(); + + NewSearcherListener trigger = new NewSearcherListener(); + core.registerNewSearcherListener(trigger); + DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler(); + CommitTracker tracker = updater.commitTracker; + tracker.setTimeUpperBound(0); + tracker.setDocsUpperBound(-1); + + UpdateRequestHandler handler = new UpdateRequestHandler(); + handler.init( null ); + + MapSolrParams params = new MapSolrParams( new HashMap() ); + + // Add a single document with commitWithin == 2 second + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrQueryRequestBase req = new SolrQueryRequestBase( core, params ) {}; + req.setContentStreams( AutoCommitTest.toContentStreams( + adoc(2000, "id", "529", "field_t", "what's inside?", "subject", "info"), null ) ); + trigger.reset(); + handler.handleRequest( req, rsp ); + + // Check it isn't in the index + assertQ("shouldn't find any", req("id:529") ,"//result[@numFound=0]" ); + + // Wait longer than the commitWithin time + assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000)); + + // Add one document without commitWithin + req.setContentStreams( AutoCommitTest.toContentStreams( + adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null ) ); + trigger.reset(); + handler.handleRequest( req, rsp ); + + // Check it isn't in the index + assertQ("shouldn't find any", req("id:530") ,"//result[@numFound=0]" ); + + // Delete one document with commitWithin + req.setContentStreams( AutoCommitTest.toContentStreams( + delI("529", "commitWithin", "1000"), null ) ); + trigger.reset(); + handler.handleRequest( req, rsp ); + + // Now make sure we can find it + assertQ("should find one", req("id:529") ,"//result[@numFound=1]" ); + + // Wait for the commit to happen + assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000)); + + // Now we shouldn't find it + assertQ("should find none", req("id:529") ,"//result[@numFound=0]" ); + // ... but we should find the new one + assertQ("should find one", req("id:530") ,"//result[@numFound=1]" ); + + trigger.reset(); + + // now make the call 10 times really fast and make sure it + // only commits once + req.setContentStreams( AutoCommitTest.toContentStreams( + adoc(2000, "id", "500" ), null ) ); + for( int i=0;i<10; i++ ) { + handler.handleRequest( req, rsp ); + } + assertQ("should not be there yet", req("id:500") ,"//result[@numFound=0]" ); + + // the same for the delete + req.setContentStreams( AutoCommitTest.toContentStreams( + delI("530", "commitWithin", "1000"), null ) ); + for( int i=0;i<10; i++ ) { + handler.handleRequest( req, rsp ); + } + assertQ("should be there", req("id:530") ,"//result[@numFound=1]" ); + + assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000)); + assertQ("should be there", req("id:500") ,"//result[@numFound=1]" ); + assertQ("should not be there", req("id:530") ,"//result[@numFound=0]" ); + + assertEquals(3, tracker.getCommitCount()); + } + +}