From commits-return-100817-archive-asf-public=cust-asf.ponee.io@lucene.apache.org Tue May 8 23:14:59 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 9EC80180674 for ; Tue, 8 May 2018 23:14:55 +0200 (CEST) Received: (qmail 17207 invoked by uid 500); 8 May 2018 21:14:49 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 16451 invoked by uid 99); 8 May 2018 21:14:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 May 2018 21:14:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E124AF6C82; Tue, 8 May 2018 21:14:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ab@apache.org To: commits@lucene.apache.org Date: Tue, 08 May 2018 21:15:19 -0000 Message-Id: <2815a2e55470431db07b67c763341192@git.apache.org> In-Reply-To: <871f5f239d3a44bfbb1af4abe3af8906@git.apache.org> References: <871f5f239d3a44bfbb1af4abe3af8906@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [33/50] [abbrv] lucene-solr:jira/solr-11779: SOLR-11277: Add auto hard commit setting based on tlog size (this closes #358) SOLR-11277: Add auto hard commit setting based on tlog size (this closes #358) Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b6174896 Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b6174896 Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b6174896 Branch: refs/heads/jira/solr-11779 Commit: b617489638db4ddca63e5fbc45a58c5695a021d3 Parents: ab11867 Author: Anshum Gupta Authored: Thu May 3 15:00:47 2018 -0700 Committer: Anshum Gupta Committed: Thu May 3 15:00:47 2018 -0700 ---------------------------------------------------------------------- solr/CHANGES.txt | 2 + .../java/org/apache/solr/core/SolrConfig.java | 45 ++- .../org/apache/solr/update/CommitTracker.java | 66 ++- .../solr/update/DirectUpdateHandler2.java | 45 ++- .../org/apache/solr/update/TransactionLog.java | 7 + .../java/org/apache/solr/update/UpdateLog.java | 7 + .../conf/bad-solrconfig-no-autocommit-tag.xml | 52 +++ .../solr/collection1/conf/solrconfig-tlog.xml | 4 + .../test/org/apache/solr/core/TestConfig.java | 36 ++ .../solr/update/MaxSizeAutoCommitTest.java | 398 +++++++++++++++++++ 10 files changed, 642 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/CHANGES.txt ---------------------------------------------------------------------- diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b925fe1..d4c2097 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -110,6 +110,8 @@ New Features * SOLR-11278: Add IgnoreLargeDocumentProcessFactory (Cao Manh Dat, David Smiley) +* SOLR-11277: Add auto hard-commit settings based on tlog size (Rupa Shankar, Anshum Gupta) + Bug Fixes ---------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/core/SolrConfig.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 6c67645..6bfa08d 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -54,6 +54,7 @@ import java.util.regex.Pattern; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathConstants; +import org.apache.commons.io.FileUtils; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.Version; @@ -448,6 +449,7 @@ public class SolrConfig extends Config implements MapSerializable { return new UpdateHandlerInfo(get("updateHandler/@class", null), getInt("updateHandler/autoCommit/maxDocs", -1), getInt("updateHandler/autoCommit/maxTime", -1), + convertHeapOptionStyleConfigStringToBytes(get("updateHandler/autoCommit/maxSize", "")), getBool("updateHandler/indexWriter/closeWaitsForMerges", true), getBool("updateHandler/autoCommit/openSearcher", true), getInt("updateHandler/autoSoftCommit/maxDocs", -1), @@ -455,6 +457,44 @@ public class SolrConfig extends Config implements MapSerializable { getBool("updateHandler/commitWithin/softCommit", true)); } + /** + * Converts a Java heap option-like config string to bytes. Valid suffixes are: 'k', 'm', 'g' + * (case insensitive). If there is no suffix, the default unit is bytes. + * For example, 50k = 50KB, 20m = 20MB, 4g = 4GB, 300 = 300 bytes + * @param configStr the config setting to parse + * @return the size, in bytes. -1 if the given config string is empty + */ + protected static long convertHeapOptionStyleConfigStringToBytes(String configStr) { + if (configStr.isEmpty()) { + return -1; + } + long multiplier = 1; + String numericValueStr = configStr; + char suffix = Character.toLowerCase(configStr.charAt(configStr.length() - 1)); + if (Character.isLetter(suffix)) { + if (suffix == 'k') { + multiplier = FileUtils.ONE_KB; + } + else if (suffix == 'm') { + multiplier = FileUtils.ONE_MB; + } + else if (suffix == 'g') { + multiplier = FileUtils.ONE_GB; + } else { + throw new RuntimeException("Invalid suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). " + + "No suffix means the amount is in bytes. "); + } + numericValueStr = configStr.substring(0, configStr.length() - 1); + } + try { + return Long.parseLong(numericValueStr) * multiplier; + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid format. The config setting should be a long with an " + + "optional letter suffix. Valid suffixes are 'k' (KB), 'm' (MB), 'g' (G). " + + "No suffix means the amount is in bytes."); + } + } + private void loadPluginInfo(SolrPluginInfo pluginInfo) { boolean requireName = pluginInfo.options.contains(REQUIRE_NAME); boolean requireClass = pluginInfo.options.contains(REQUIRE_CLASS); @@ -631,6 +671,7 @@ public class SolrConfig extends Config implements MapSerializable { public final String className; public final int autoCommmitMaxDocs, autoCommmitMaxTime, autoSoftCommmitMaxDocs, autoSoftCommmitMaxTime; + public final long autoCommitMaxSizeBytes; public final boolean indexWriterCloseWaitsForMerges; public final boolean openSearcher; // is opening a new searcher part of hard autocommit? public final boolean commitWithinSoftCommit; @@ -638,12 +679,14 @@ public class SolrConfig extends Config implements MapSerializable { /** * @param autoCommmitMaxDocs set -1 as default * @param autoCommmitMaxTime set -1 as default + * @param autoCommitMaxSize set -1 as default */ - public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, boolean indexWriterCloseWaitsForMerges, boolean openSearcher, + public UpdateHandlerInfo(String className, int autoCommmitMaxDocs, int autoCommmitMaxTime, long autoCommitMaxSize, boolean indexWriterCloseWaitsForMerges, boolean openSearcher, int autoSoftCommmitMaxDocs, int autoSoftCommmitMaxTime, boolean commitWithinSoftCommit) { this.className = className; this.autoCommmitMaxDocs = autoCommmitMaxDocs; this.autoCommmitMaxTime = autoCommmitMaxTime; + this.autoCommitMaxSizeBytes = autoCommitMaxSize; this.indexWriterCloseWaitsForMerges = indexWriterCloseWaitsForMerges; this.openSearcher = openSearcher; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/CommitTracker.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/update/CommitTracker.java b/solr/core/src/java/org/apache/solr/update/CommitTracker.java index 6cf7504..8f06d11 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitTracker.java +++ b/solr/core/src/java/org/apache/solr/update/CommitTracker.java @@ -49,10 +49,13 @@ public final class CommitTracker implements Runnable { // scheduler delay for maxDoc-triggered autocommits public static final int DOC_COMMIT_DELAY_MS = 1; + // scheduler delay for maxSize-triggered autocommits + public static final int SIZE_COMMIT_DELAY_MS = 1; // settings, not final so we can change them in testing private int docsUpperBound; private long timeUpperBound; + private long tLogFileSizeUpperBound; private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DefaultSolrThreadFactory("commitScheduler")); @@ -70,13 +73,15 @@ public final class CommitTracker implements Runnable { private String name; - public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, boolean openSearcher, boolean softCommit) { + public CommitTracker(String name, SolrCore core, int docsUpperBound, int timeUpperBound, long tLogFileSizeUpperBound, + boolean openSearcher, boolean softCommit) { this.core = core; this.name = name; pending = null; this.docsUpperBound = docsUpperBound; this.timeUpperBound = timeUpperBound; + this.tLogFileSizeUpperBound = tLogFileSizeUpperBound; this.softCommit = softCommit; this.openSearcher = openSearcher; @@ -154,9 +159,34 @@ public final class CommitTracker implements Runnable { /** * Indicate that documents have been added + * @param commitWithin amount of time (in ms) within which a commit should be scheduled */ public void addedDocument(int commitWithin) { - // maxDocs-triggered autoCommit. Use == instead of > so we only trigger once on the way up + addedDocument(commitWithin, -1); + } + + /** + * Indicate that documents have been added + * @param commitWithin amount of time (in ms) within which a commit should be scheduled + * @param currentTlogSize current tlog size (in bytes). Use -1 if we don't want to check for a max size triggered commit + */ + public void addedDocument(int commitWithin, long currentTlogSize) { + // maxDocs-triggered autoCommit + _scheduleMaxDocsTriggeredCommitIfNeeded(); + + // maxTime-triggered autoCommit + _scheduleCommitWithinIfNeeded(commitWithin); + + // maxSize-triggered autoCommit + _scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize); + } + + /** + * If a doc size upper bound is set, and the current number of documents has exceeded it, then + * schedule a commit and reset the counter + */ + private void _scheduleMaxDocsTriggeredCommitIfNeeded() { + // Use == instead of > so we only trigger once on the way up if (docsUpperBound > 0) { long docs = docsSinceCommit.incrementAndGet(); if (docs == docsUpperBound + 1) { @@ -165,9 +195,6 @@ public final class CommitTracker implements Runnable { _scheduleCommitWithin(DOC_COMMIT_DELAY_MS); } } - - // maxTime-triggered autoCommit - _scheduleCommitWithinIfNeeded(commitWithin); } /** @@ -176,6 +203,26 @@ public final class CommitTracker implements Runnable { public void deletedDocument( int commitWithin ) { _scheduleCommitWithinIfNeeded(commitWithin); } + + /** + * If the given current tlog size is greater than the file size upper bound, then schedule a commit + * @param currentTlogSize current tlog size (in bytes) + */ + public void scheduleMaxSizeTriggeredCommitIfNeeded(long currentTlogSize) { + _scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize); + } + + /** + * If the given current tlog size is greater than the file size upper bound, then schedule a commit + * @param currentTlogSize current tlog size (in bytes) + */ + private void _scheduleMaxSizeTriggeredCommitIfNeeded(long currentTlogSize) { + if (tLogFileSizeUpperBound > 0 && currentTlogSize > tLogFileSizeUpperBound) { + docsSinceCommit.set(0); + _scheduleCommitWithin(SIZE_COMMIT_DELAY_MS); + } + } + /** Inform tracker that a commit has occurred */ public void didCommit() { @@ -254,6 +301,10 @@ public final class CommitTracker implements Runnable { return docsUpperBound; } + long getTLogFileSizeUpperBound() { + return tLogFileSizeUpperBound; + } + void setDocsUpperBound(int docsUpperBound) { this.docsUpperBound = docsUpperBound; } @@ -262,6 +313,11 @@ public final class CommitTracker implements Runnable { public void setTimeUpperBound(long timeUpperBound) { this.timeUpperBound = timeUpperBound; } + + // only for testing - not thread safe + public void setTLogFileSizeUpperBound(int sizeUpperBound) { + this.tLogFileSizeUpperBound = sizeUpperBound; + } // only for testing - not thread safe public void setOpenSearcher(boolean openSearcher) { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 922419c..db12dc3 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -73,6 +73,9 @@ import org.slf4j.LoggerFactory; * directly to the main Lucene index as opposed to adding to a separate smaller index. */ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState.IndexWriterCloser, SolrMetricProducer { + + private static final int NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER = -1; + protected final SolrCoreState solrCoreState; // stats @@ -118,13 +121,14 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig() .getUpdateHandlerInfo(); - int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1); - int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1); - commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false); + int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; + int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; + long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes; + commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, fileSizeUpperBound, updateHandlerInfo.openSearcher, false); - int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1); - int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1); - softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, true, true); + int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; + int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; + softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER, true, true); commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges; @@ -143,13 +147,14 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig() .getUpdateHandlerInfo(); - int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; // getInt("updateHandler/autoCommit/maxDocs", -1); - int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; // getInt("updateHandler/autoCommit/maxTime", -1); - commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, updateHandlerInfo.openSearcher, false); + int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs; + int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime; + long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes; + commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, fileSizeUpperBound, updateHandlerInfo.openSearcher, false); - int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; // getInt("updateHandler/autoSoftCommit/maxDocs", -1); - int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; // getInt("updateHandler/autoSoftCommit/maxTime", -1); - softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, updateHandlerInfo.openSearcher, true); + int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs; + int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime; + softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER, updateHandlerInfo.openSearcher, true); commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit; indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges; @@ -178,6 +183,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState manager.registerGauge(this, registryName, () -> "" + commitTracker.getTimeUpperBound() + "ms", tag, true, "autoCommitMaxTime", getCategory().toString(), scope); } + if (commitTracker.getTLogFileSizeUpperBound() > 0) { + manager.registerGauge(this, registryName, () -> commitTracker.getTLogFileSizeUpperBound(), tag, true, "autoCommitMaxSize", + getCategory().toString(), scope); + } if (softCommitTracker.getDocsUpperBound() > 0) { manager.registerGauge(this, registryName, () -> softCommitTracker.getDocsUpperBound(), tag, true, "softAutoCommitMaxDocs", getCategory().toString(), scope); @@ -279,12 +288,13 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } if ((cmd.getFlags() & UpdateCommand.IGNORE_AUTOCOMMIT) == 0) { + long currentTlogSize = getCurrentTLogSize(); if (commitWithinSoftCommit) { - commitTracker.addedDocument(-1); + commitTracker.addedDocument(-1, currentTlogSize); softCommitTracker.addedDocument(cmd.commitWithin); } else { softCommitTracker.addedDocument(-1); - commitTracker.addedDocument(cmd.commitWithin); + commitTracker.addedDocument(cmd.commitWithin, currentTlogSize); } } @@ -418,6 +428,9 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState if (commitTracker.getTimeUpperBound() > 0) { commitTracker.scheduleCommitWithin(commitTracker.getTimeUpperBound()); } + + long currentTlogSize = getCurrentTLogSize(); + commitTracker.scheduleMaxSizeTriggeredCommitIfNeeded(currentTlogSize); if (softCommitTracker.getTimeUpperBound() > 0) { softCommitTracker.scheduleCommitWithin(softCommitTracker @@ -990,6 +1003,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState return solrCoreState; } + private long getCurrentTLogSize() { + return ulog != null && ulog.hasUncommittedChanges() ? ulog.getCurrentLogSizeFromStream() : -1; + } + // allow access for tests public CommitTracker getCommitTracker() { return commitTracker; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/TransactionLog.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java index be4dabc..96a928c 100644 --- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java +++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java @@ -630,6 +630,13 @@ public class TransactionLog implements Closeable { return 0; } + /** + * @return the FastOutputStream size + */ + public synchronized long getLogSizeFromStream() { + return fos.size(); + } + /** Returns a reader that can be used while a log is still in use. * Currently only *one* LogReader may be outstanding, and that log may only * be used from a single thread. */ http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/java/org/apache/solr/update/UpdateLog.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index fbdf616..09ff146 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -292,6 +292,13 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer { return size; } + /** + * @return the current transaction log's size (based on its output stream) + */ + public long getCurrentLogSizeFromStream() { + return tlog.getLogSizeFromStream(); + } + public long getTotalLogsNumber() { synchronized (this) { return logs.size(); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml ---------------------------------------------------------------------- diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml b/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml new file mode 100644 index 0000000..1040db6 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-solrconfig-no-autocommit-tag.xml @@ -0,0 +1,52 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + 5k + + + + + + + + explicit + true + text + + + + + http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml ---------------------------------------------------------------------- diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml index c8884c6..989023d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml @@ -54,6 +54,10 @@ + + + ${solr.autoCommit.maxSize:} + ${solr.ulog.dir:} ${solr.ulog.maxNumLogsToKeep:10} http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test/org/apache/solr/core/TestConfig.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java index 5a7b706..aa43c25 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java @@ -30,6 +30,7 @@ import org.apache.solr.handler.admin.ShowFileRequestHandler; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchemaFactory; import org.apache.solr.update.SolrIndexConfig; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.w3c.dom.Node; @@ -181,6 +182,41 @@ public class TestConfig extends SolrTestCaseJ4 { assertEquals("numDefaultsTested vs. numDefaultsMapped+numNullDefaults ="+sic.toMap(new LinkedHashMap<>()).keySet(), numDefaultsTested, numDefaultsMapped+numNullDefaults); } + @Test + public void testConvertAutoCommitMaxSizeStringToBytes() { + + // Valid values + Assert.assertEquals(300, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300")); + Assert.assertEquals(307200, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300k")); + Assert.assertEquals(307200, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300K")); + Assert.assertEquals(314572800, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300m")); + Assert.assertEquals(314572800, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300M")); + Assert.assertEquals(322122547200L, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300g")); + Assert.assertEquals(322122547200L, SolrConfig.convertHeapOptionStyleConfigStringToBytes("300G")); + Assert.assertEquals(-1, SolrConfig.convertHeapOptionStyleConfigStringToBytes("")); + + // Invalid values + try { + SolrConfig.convertHeapOptionStyleConfigStringToBytes("3jbk32k"); // valid suffix but non-numeric prefix + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Invalid")); + } + try { + SolrConfig.convertHeapOptionStyleConfigStringToBytes("300x"); // valid prefix but invalid suffix + Assert.fail(); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Invalid")); + } + } + + @Test + public void testMaxSizeSettingWithoutAutoCommit() throws Exception { + SolrConfig solrConfig = new SolrConfig(new SolrResourceLoader(TEST_PATH().resolve("collection1")), "bad-solrconfig-no-autocommit-tag.xml", null); + Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommitMaxSizeBytes); + Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommmitMaxDocs); + Assert.assertEquals(-1, solrConfig.getUpdateHandlerInfo().autoCommmitMaxTime); + } // sanity check that sys propertis are working as expected public void testSanityCheckTestSysPropsAreUsed() throws Exception { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6174896/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java new file mode 100644 index 0000000..0db3616 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/MaxSizeAutoCommitTest.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import com.carrotsearch.randomizedtesting.annotations.Repeat; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class MaxSizeAutoCommitTest extends SolrTestCaseJ4 { + + // Given an ID, returns an XML string for an "add document" request + private static final Function ADD_DOC_FN = (id) -> adoc("id", Integer.toString(id)); + // Given an ID, returns an XML string for a "delete document" request + private static final Function DELETE_DOC_FN = (id) -> delI(Integer.toString(id)); + + private ObjectMapper objectMapper; // for JSON parsing + private SolrCore core; + private DirectUpdateHandler2 updateHandler; + private CommitTracker hardCommitTracker; + private UpdateRequestHandler updateRequestHandler; + private String tlogDirPath; + + @Before + public void setup() throws Exception { + objectMapper = new ObjectMapper(); + System.setProperty("solr.autoCommit.maxSize", "5k"); + System.setProperty("solr.ulog", "solr.UpdateLog"); + initCore("solrconfig-tlog.xml", "schema.xml"); + core = h.getCore(); + updateHandler = (DirectUpdateHandler2) core.getUpdateHandler(); + hardCommitTracker = updateHandler.commitTracker; + // Only testing file-size based auto hard commits - disable other checks + hardCommitTracker.setTimeUpperBound(-1); + hardCommitTracker.setDocsUpperBound(-1); + updateRequestHandler = new UpdateRequestHandler(); + updateRequestHandler.init( null ); + tlogDirPath = core.getDataDir() + "/tlog"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + System.clearProperty("solr.autoCommit.maxSize"); + System.clearProperty("solr.ulog"); + deleteCore(); + } + + @Test + public void simpleTest() throws Exception { + int maxFileSizeBound = 1000; + int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25); + // Set max size bound + hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound); + + // Adding these docs will place the tlog size just under the threshold + int numDocs = 27; + int batchSize = 3; + int numBatches = numDocs / batchSize; + SolrQueryResponse updateResp = new SolrQueryResponse(); + int numTlogs = -1; + TreeMap tlogsInfo = null; + + for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) { + int docStartId = batchSize * batchCounter; + + // Send batch update request + updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp); + + // The sleep is to allow existing commits to finish (or at least mostly finish) before querying/submitting more documents + waitForCommit(200); + + // There should just be 1 tlog and its size should be within the (buffered) file size bound + tlogsInfo = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer); + numTlogs = parseTotalNumTlogs(tlogsInfo); + Assert.assertEquals(1, numTlogs); + } + + // Now that the core's tlog size is just under the threshold, one more update should induce a commit + int docStartId = batchSize * numBatches; + updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp); + waitForCommit(200); + + // Verify that a commit happened. There should now be 2 tlogs, both of which are < maxFileSizeBound. + TreeMap tlogsInfoPostCommit = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer); + Assert.assertEquals(2, parseTotalNumTlogs(tlogsInfoPostCommit)); + + // And the current tlog's size should be less than the previous tlog's size + Assert.assertTrue(tlogsInfoPostCommit.lastEntry().getValue() < tlogsInfo.lastEntry().getValue()); + } + + @Test + public void testRedundantDeletes() throws Exception { + int maxFileSizeBound = 1000; + int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25); + + // Set max size bound + hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound); + + // Add docs + int numDocsToAdd = 150; + SolrQueryResponse updateResp = new SolrQueryResponse(); + updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp); + waitForCommit(200); + + // Get the tlog file info + TreeMap tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath); + + // Send a bunch of redundant deletes + int numDeletesToSend = 5000; + int docIdToDelete = 100; + + SolrQueryRequestBase requestWithOneDelete = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap())) {}; + List docs = new ArrayList<>(); + docs.add(delI(Integer.toString(docIdToDelete))); + + requestWithOneDelete.setContentStreams(toContentStreams(docs)); + + for (int i = 0; i < numDeletesToSend; i++) { + if (i % 50 == 0) { + // Wait periodically to allow existing commits to finish before + // sending more delete requests + waitForCommit(200); + } + updateRequestHandler.handleRequest(requestWithOneDelete, updateResp); + } + + // Verify that new tlogs have been created, and that their sizes are as expected + TreeMap tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer); + Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes)); + } + + @Test + public void deleteTest() throws Exception { + int maxFileSizeBound = 1000; + int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.25); + + // Set max size bound + hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound); + + // Add docs + int numDocsToAdd = 150; + SolrQueryResponse updateResp = new SolrQueryResponse(); + updateRequestHandler.handleRequest(constructBatchAddDocRequest(0, numDocsToAdd), updateResp); + waitForCommit(200); + + // Get the tlog file info + TreeMap tlogsInfoPreDeletes = getTlogFileSizes(tlogDirPath); + + // Delete documents (in batches, so we can allow commits to finish and new tlog files to be created) + int batchSize = 15; + int numBatches = numDocsToAdd / batchSize; + for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) { + int docStartId = batchSize * batchCounter; + + // Send batch delete doc request + updateRequestHandler.handleRequest(constructBatchDeleteDocRequest(docStartId, batchSize), updateResp); + + // The sleep is to allow existing commits to finish before deleting more documents + waitForCommit(200); + } + + // Verify that the commit happened by seeing if a new tlog file was opened + TreeMap tlogsInfoPostDeletes = getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer); + Assert.assertTrue(parseTotalNumTlogs(tlogsInfoPreDeletes) < parseTotalNumTlogs(tlogsInfoPostDeletes)); + } + + @Test + @Repeat(iterations = 5) + public void endToEndTest() throws Exception { + int maxFileSizeBound = 5000; + // Set max size bound + hardCommitTracker.setTLogFileSizeUpperBound(maxFileSizeBound); + + // Giving a 10% buffer for the max size bound + int maxFileSizeBoundWithBuffer = (int) (maxFileSizeBound * 1.1); + + SolrQueryRequest selectQuery = req("*:*"); + List docCounts = new ArrayList<>(); + + int numDocs = 1000; + int batchSize = 20; + int numBatches = numDocs / batchSize; + for (int batchCounter = 0; batchCounter < numBatches; batchCounter++) { + SolrQueryResponse updateResp = new SolrQueryResponse(); + int docStartId = batchSize * batchCounter; + + // Send batch add doc request + updateRequestHandler.handleRequest(constructBatchAddDocRequest(docStartId, batchSize), updateResp); + + // The sleep is to allow existing commits to finish before querying/submitting more documents + waitForCommit(200); + + // Check tlog file sizes + getTlogFileSizes(tlogDirPath, maxFileSizeBoundWithBuffer); + + // See how many documents are currently visible. This should increase as more commits occur. + docCounts.add(queryCore(selectQuery)); + } + + // One final commit, after which all documents should be visible + CommitUpdateCommand commitUpdateCommand = new CommitUpdateCommand(req(), false); + updateHandler.commit(commitUpdateCommand); + waitForCommit(200); + docCounts.add(queryCore(selectQuery)); + + // Evaluate the document counts + checkNumFoundDocuments(docCounts, numDocs); + } + + /** + * Sleeps in increments of 50 ms while checking to see if a commit completed. If it did, then return. If not, continue + * this cycle for at most the amount of time specified + * @param maxTotalWaitTimeMillis the max amount of time (in ms) to wait/check for a commit + */ + private void waitForCommit(long maxTotalWaitTimeMillis) throws Exception { + long startTimeNanos = System.nanoTime(); + long maxTotalWaitTimeNanos = TimeUnit.MILLISECONDS.toNanos(maxTotalWaitTimeMillis); + while (System.nanoTime() - startTimeNanos < maxTotalWaitTimeNanos) { + Thread.sleep(50); + if (!updateHandler.getUpdateLog().hasUncommittedChanges()) { + return; + } + } + } + + /** + * Returns the total number of tlogs that have been created for the core. + * + * The tlogs in a core's tlog directory are named: tlog.0000000000000000000, tlog.0000000000000000001, tlog.0000000000000000002, etc. + * Because old tlogs are periodically deleted, we can't just count the number of existing files. Instead, we take the + * highest ordering tlog file name (which would be the newest) and parse the extension. + * + * e.g if the most recently created tlog file is tlog.0000000000000000003, we know that this core has had 4 tlogs. + * + * @param tlogsInfo TreeMap of (tlog file name, tlog file size (in bytes)) pairs + * @return total number of tlogs created for this core + */ + private int parseTotalNumTlogs(TreeMap tlogsInfo) { + String mostRecentFileName = tlogsInfo.lastKey(); + int extensionDelimiterIndex = mostRecentFileName.lastIndexOf("."); + if (extensionDelimiterIndex == -1) { + throw new RuntimeException("Invalid tlog filename: " + mostRecentFileName); + } + String extension = mostRecentFileName.substring(extensionDelimiterIndex + 1); + try { + return Integer.parseInt(extension) + 1; + } catch (NumberFormatException e) { + throw new RuntimeException("Could not parse tlog filename: " + mostRecentFileName, e); + } + } + + /** + * Construct a batch add document request with a series of very simple Solr docs with increasing IDs. + * @param startId the document ID to begin with + * @param batchSize the number of documents to include in the batch + * @return a SolrQueryRequestBase + */ + private SolrQueryRequestBase constructBatchAddDocRequest(int startId, int batchSize) { + return constructBatchRequestHelper(startId, batchSize, ADD_DOC_FN); + } + + /** + * Construct a batch delete document request, with IDs incrementing from startId + * @param startId the document ID to begin with + * @param batchSize the number of documents to include in the batch + * @return a SolrQueryRequestBase + */ + private SolrQueryRequestBase constructBatchDeleteDocRequest(int startId, int batchSize) { + return constructBatchRequestHelper(startId, batchSize, DELETE_DOC_FN); + } + + /** + * Helper for constructing a batch update request + * @param startId the document ID to begin with + * @param batchSize the number of documents to include in the batch + * @param requestFn a function that takes an (int) ID and returns an XML string of the request to add to the batch request + * @return a SolrQueryRequestBase + */ + private SolrQueryRequestBase constructBatchRequestHelper(int startId, int batchSize, Function requestFn) { + SolrQueryRequestBase updateReq = new SolrQueryRequestBase(core, new MapSolrParams(new HashMap<>())) {}; + List docs = new ArrayList<>(); + for (int i = startId; i < startId + batchSize; i++) { + docs.add(requestFn.apply(i)); + } + updateReq.setContentStreams(toContentStreams(docs)); + return updateReq; + } + + /** + * Executes the given query + * @param query the query to execute + * @return the number of documents found + */ + public int queryCore(SolrQueryRequest query) throws Exception { + String responseStr = h.query(query); + try { + Map root = (Map) objectMapper.readValue(responseStr, Object.class); + Map rootResponse = (Map) root.get("response"); + return (int) rootResponse.get("numFound"); + } catch (Exception e) { + throw new RuntimeException("Unable to parse Solr query response", e); + } + } + + /** + * Checks the given list of document counts to make sure that they are increasing (as commits occur). + * @param numDocList list of the number of documents found in a given core. Ascending from oldest to newest + */ + private void checkNumFoundDocuments(List numDocList, int finalValue) { + long currentTotal = 0; + for (Integer count : numDocList) { + Assert.assertTrue(count >= currentTotal); + currentTotal = count; + } + Assert.assertEquals(finalValue, numDocList.get(numDocList.size() - 1).intValue()); + } + + + /** + * Goes through the given tlog directory and inspects each tlog. + * @param tlogDirPath tlog directory path + * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs + */ + private TreeMap getTlogFileSizes(String tlogDirPath) { + return getTlogFileSizes(tlogDirPath, Integer.MAX_VALUE); + } + + /** + * Goes through the given tlog directory and inspects each tlog. Asserts that each tlog's size is <= the given max size bound. + * @param tlogDirPath tlog directory path + * @param maxSizeBound the max tlog size + * @return a TreeMap of (tlog file name, tlog file size (in bytes)) pairs + */ + private TreeMap getTlogFileSizes(String tlogDirPath, int maxSizeBound) { + File tlogDir = new File(tlogDirPath); + File[] tlogs = tlogDir.listFiles(); + TreeMap tlogInfo = new TreeMap<>(); + if (tlogs != null) { + for (File tlog : tlogs) { + String message = String.format(Locale.getDefault(), "Tlog size exceeds the max size bound. Tlog path: %s, tlog size: %d", + tlog.getPath(), tlog.length()); + Assert.assertTrue(message, tlog.length() <= maxSizeBound); + tlogInfo.put(tlog.getName(), tlog.length()); + } + } + return tlogInfo; + } + + /** + * Convert the given list of strings into a list of streams, for Solr update requests + * @param strs strings to convert into streams + * @return list of streams + */ + private List toContentStreams(List strs) { + ArrayList streams = new ArrayList<>(); + for (String str : strs) { + streams.addAll(ClientUtils.toContentStreams(str, "text/xml")); + } + return streams; + } +}