lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject lucene-solr:branch_6_0: SOLR-9036: Solr slave is doing full replication (entire index) of index after master restart (cherry picked from commit 51b1319)
Date Fri, 20 May 2016 21:36:49 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6_0 a6adc681b -> 93a33e089


SOLR-9036: Solr slave is doing full replication (entire index) of index after master restart
(cherry picked from commit 51b1319)


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/93a33e08
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/93a33e08
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/93a33e08

Branch: refs/heads/branch_6_0
Commit: 93a33e0892e2ad3c8f73755d4ba10da72ff9c11a
Parents: a6adc68
Author: Shalin Shekhar Mangar <shalin@apache.org>
Authored: Thu May 5 21:00:20 2016 +0530
Committer: Steve Rowe <sarowe@apache.org>
Committed: Fri May 20 17:33:35 2016 -0400

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  3 +
 .../org/apache/solr/handler/IndexFetcher.java   |  5 +-
 .../solr/handler/TestReplicationHandler.java    | 77 ++++++++++++++++++++
 3 files changed, 84 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93a33e08/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6feee5d..dbadf9a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -66,6 +66,9 @@ Bug Fixes
 * SOLR-9030: The 'downnode' overseer command can trip asserts in ZkStateWriter.
   (Scott Blum, Mark Miller, shalin)
 
+* SOLR-9036: Solr slave is doing full replication (entire index) of index after master restart.
+  (Lior Sapir, Mark Miller, shalin)
+
 Other Changes
 ----------------------
 * SOLR-7516: Improve javadocs for JavaBinCodec, ObjectResolver and enforce the single-usage
policy.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93a33e08/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 39c4158..c172f81 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -570,7 +570,10 @@ public class IndexFetcher {
         }
       }
 
-      core.getUpdateHandler().getSolrCoreState().setLastReplicateIndexSuccess(successfulInstall);
+      if (core.getCoreDescriptor().getCoreContainer().isZooKeeperAware()) {
+        // we only track replication success in SolrCloud mode
+        core.getUpdateHandler().getSolrCoreState().setLastReplicateIndexSuccess(successfulInstall);
+      }
 
       filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = tlogFilesToDownload
= tlogFilesDownloaded = null;
       markReplicationStop();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93a33e08/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index eab3e87..526dabe 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -58,6 +58,7 @@ import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -582,6 +583,82 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
     assertEquals(nDocs+1, numFound(rQuery(nDocs+1, "*:*", slaveClient)));
   }
 
+  /**
+   * We assert that if master is down for more than poll interval,
+   * the slave doesn't re-fetch the whole index from master again if
+   * the index hasn't changed. See SOLR-9036
+   */
+  @Test
+  public void doTestIndexFetchOnMasterRestart() throws Exception  {
+    useFactory(null);
+    try {
+      clearIndexWithReplication();
+      // change solrconfig having 'replicateAfter startup' option on master
+      master.copyConfigFile(CONF_DIR + "solrconfig-master2.xml",
+          "solrconfig.xml");
+
+      masterJetty.stop();
+      masterJetty.start();
+
+      nDocs--;
+      for (int i = 0; i < nDocs; i++)
+        index(masterClient, "id", i, "name", "name = " + i);
+
+      masterClient.commit();
+
+      NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
+      SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
+      assertEquals(nDocs, numFound(masterQueryRsp));
+
+      //get docs from slave and check if number is equal to master
+      NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
+      SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
+      assertEquals(nDocs, numFound(slaveQueryRsp));
+
+      //compare results
+      String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult,
0, null);
+      assertEquals(null, cmp);
+
+      assertEquals(1, Integer.parseInt(getSlaveDetails("timesIndexReplicated")));
+      String timesFailed = getSlaveDetails("timesFailed");
+      assertEquals(0, Integer.parseInt(timesFailed != null ?  timesFailed : "0"));
+
+      masterJetty.stop();
+
+      // poll interval on slave is 1 second, so we just sleep for a few seconds
+      Thread.sleep(2000);
+
+      masterJetty.start();
+
+      // poll interval on slave is 1 second, so we just sleep for a few seconds
+      Thread.sleep(2000);
+
+      //get docs from slave and assert that they are still the same as before
+      slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
+      slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
+      assertEquals(nDocs, numFound(slaveQueryRsp));
+
+      int failed = Integer.parseInt(getSlaveDetails("timesFailed"));
+      assertTrue(failed > 0);
+      assertEquals(1, Integer.parseInt(getSlaveDetails("timesIndexReplicated")) - failed);
+    } finally {
+      resetFactory();
+    }
+  }
+
+  private String getSlaveDetails(String keyName) throws SolrServerException, IOException
{
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set(CommonParams.QT, "/replication");
+    params.set("command", "details");
+    QueryResponse response = slaveClient.query(params);
+    System.out.println("SHALIN: " + response.getResponse());
+    // details/slave/timesIndexReplicated
+    NamedList<Object> details = (NamedList<Object>) response.getResponse().get("details");
+    NamedList<Object> slave = (NamedList<Object>) details.get("slave");
+    Object o = slave.get(keyName);
+    return o != null ? o.toString() : null;
+  }
+
   @Test
   public void doTestIndexFetchWithMasterUrl() throws Exception {
     //change solrconfig on slave


Mime
View raw message