From commits-return-99968-archive-asf-public=cust-asf.ponee.io@lucene.apache.org Mon Apr 2 09:37:57 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id BF85E18067C for ; Mon, 2 Apr 2018 09:37:56 +0200 (CEST) Received: (qmail 77984 invoked by uid 500); 2 Apr 2018 07:37:54 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 72149 invoked by uid 99); 2 Apr 2018 07:37:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 02 Apr 2018 07:37:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 66574F6AD6; Mon, 2 Apr 2018 07:37:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: shalin@apache.org To: commits@lucene.apache.org Date: Mon, 02 Apr 2018 07:38:18 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [31/34] lucene-solr:jira/solr-12095: SOLR-11673: Slave doesn't commit empty index if new index appears on master by deafult. SOLR-11673: Slave doesn't commit empty index if new index appears on master by deafult. Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a920cb0 Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a920cb0 Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a920cb0 Branch: refs/heads/jira/solr-12095 Commit: 7a920cb0f64ee3de058a980e7564685cdc311c06 Parents: d98138a Author: Mikhail Khludnev Authored: Sun Apr 1 16:47:23 2018 +0300 Committer: Mikhail Khludnev Committed: Sun Apr 1 16:47:23 2018 +0300 ---------------------------------------------------------------------- solr/CHANGES.txt | 7 ++++ .../org/apache/solr/handler/IndexFetcher.java | 35 ++++++++++++++++---- .../org/apache/solr/util/TestInjection.java | 15 +++++++++ .../solr/handler/TestReplicationHandler.java | 7 ++-- 4 files changed, 55 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/CHANGES.txt ---------------------------------------------------------------------- diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e7349cf..0dfb90b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -61,6 +61,10 @@ Upgrade Notes This is a server side change only and clients using SolrJ won't need any changes. Clients can still use any logging implementation which is compatible with SLF4J. +* SOLR-11673: Slave doesn't commit empty index when completely new index is detected on master during replication. + To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication + handler configuration, or pass it to the fetchindex command. + New Features ---------------------- @@ -95,6 +99,9 @@ Bug Fixes document. Multithreaded test for AtomicUpdateRequestProcessor was also beefed up and fixed. (Ishan Chattopadhyaya, Noble Paul, Amrit Sarkar, shalin) +* SOLR-11673: By default slave doesn't commit empty index when completely new index appears on master. + See Upgrade Notes to find a way to get back to the previous behavior. (Mikhail Khludnev) + Optimizations ---------------------- http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 2476a81..3c6859b 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -16,6 +16,31 @@ */ package org.apache.solr.handler; +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.handler.ReplicationHandler.ALIAS; +import static org.apache.solr.handler.ReplicationHandler.CHECKSUM; +import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS; +import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE; +import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST; +import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION; +import static org.apache.solr.handler.ReplicationHandler.COMMAND; +import static org.apache.solr.handler.ReplicationHandler.COMPRESSION; +import static org.apache.solr.handler.ReplicationHandler.CONF_FILES; +import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT; +import static org.apache.solr.handler.ReplicationHandler.EXTERNAL; +import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER; +import static org.apache.solr.handler.ReplicationHandler.FILE; +import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; +import static org.apache.solr.handler.ReplicationHandler.GENERATION; +import static org.apache.solr.handler.ReplicationHandler.INTERNAL; +import static org.apache.solr.handler.ReplicationHandler.MASTER_URL; +import static org.apache.solr.handler.ReplicationHandler.OFFSET; +import static org.apache.solr.handler.ReplicationHandler.SIZE; +import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO; +import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE; +import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES; + import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -86,7 +111,7 @@ import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.ReplicationHandler.*; +import org.apache.solr.handler.ReplicationHandler.FileInfo; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.SolrIndexSearcher; @@ -99,13 +124,10 @@ import org.apache.solr.util.FileUtils; import org.apache.solr.util.PropertiesOutputStream; import org.apache.solr.util.RTimer; import org.apache.solr.util.RefCounted; +import org.apache.solr.util.TestInjection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.params.CommonParams.JAVABIN; -import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.handler.ReplicationHandler.*; - import com.google.common.base.Strings; /** @@ -167,7 +189,7 @@ public class IndexFetcher { private boolean downloadTlogFiles = false; - private boolean skipCommitOnMasterVersionZero; + private boolean skipCommitOnMasterVersionZero = true; private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock"; @@ -453,6 +475,7 @@ public class IndexFetcher { } finally { iw.decref(); } + assert TestInjection.injectDelayBeforeSlaveCommitRefresh(); if (skipCommitOnMasterVersionZero) { openNewSearcherAndUpdateCommitPoint(); } else { http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/util/TestInjection.java ---------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java index 422de73..821b37e 100644 --- a/solr/core/src/java/org/apache/solr/util/TestInjection.java +++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java @@ -144,6 +144,8 @@ public class TestInjection { private static AtomicInteger countPrepRecoveryOpPauseForever = new AtomicInteger(0); + public static Integer delayBeforeSlaveCommitRefresh=null; + public static void reset() { nonGracefullClose = null; failReplicaRequests = null; @@ -158,6 +160,7 @@ public class TestInjection { waitForReplicasInSync = "true:60"; failIndexFingerprintRequests = null; wrongIndexFingerprint = null; + delayBeforeSlaveCommitRefresh = null; for (Timer timer : timers) { timer.cancel(); @@ -455,4 +458,16 @@ public class TestInjection { return new Pair<>(Boolean.parseBoolean(val), Integer.parseInt(percent)); } + public static boolean injectDelayBeforeSlaveCommitRefresh() { + if (delayBeforeSlaveCommitRefresh!=null) { + try { + log.info("Pausing IndexFetcher for {}ms", delayBeforeSlaveCommitRefresh); + Thread.sleep(delayBeforeSlaveCommitRefresh); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return true; + } + } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index ddc5b60..e8caf99 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -70,6 +70,7 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.StandardDirectoryFactory; import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.util.FileUtils; +import org.apache.solr.util.TestInjection; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -494,8 +495,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { } @Test - @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-11673") public void doTestIndexAndConfigReplication() throws Exception { + + TestInjection.delayBeforeSlaveCommitRefresh = random().nextInt(10); + clearIndexWithReplication(); nDocs--; @@ -550,7 +553,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { slaveJetty = createJetty(slave); slaveClient.close(); slaveClient = createNewSolrClient(slaveJetty.getLocalPort()); - //add a doc with new field and commit on master to trigger index fetch from slave. index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000); masterClient.commit(); @@ -567,7 +569,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { checkForSingleIndex(masterJetty); checkForSingleIndex(slaveJetty, true); - } @Test