lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject [13/50] [abbrv] lucene-solr:jira/solr-11990: SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for faster recovery
Date Sat, 28 Jul 2018 04:49:36 GMT
SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for
faster recovery


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8c06dec1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8c06dec1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8c06dec1

Branch: refs/heads/jira/solr-11990
Commit: 8c06dec16dd711c097f4f8eaa7d9df041e0cb563
Parents: 6e3f61f
Author: Cao Manh Dat <datcm@apache.org>
Authored: Mon Jul 23 09:33:28 2018 +0700
Committer: Cao Manh Dat <datcm@apache.org>
Committed: Mon Jul 23 09:33:28 2018 +0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  3 ++
 .../processor/DistributedUpdateProcessor.java   | 19 +++++--
 .../DistributedUpdateProcessorTest.java         | 54 ++++++++++++++++++++
 3 files changed, 72 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e6d468c..e93e0e7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -173,6 +173,9 @@ Optimizations
   which we can export is directly proportional to the number of sort fields specified. This
change also allows streaming
   expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
 
+* SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering
for faster recovery. 
+  (Cao Manh Dat)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 8f87510..8d715a6 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -31,6 +31,7 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.apache.solr.client.solrj.SolrRequest;
@@ -1083,7 +1084,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor
{
             // The leader forwarded us this update.
             cmd.setVersion(versionOnUpdate);
 
-            if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync ==
false) {
+            if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) {
               // we're not in an active state, and this update isn't from a replay, so buffer
it.
               cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
               ulog.add(cmd);
@@ -1132,9 +1133,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor
{
                   }
                 }
               }
-            }
-
-            if (!cmd.isInPlaceUpdate()) {
+            } else {
               // if we aren't the leader, then we need to check that updates were not re-ordered
               if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
                 // we're OK... this update has a version higher than anything we've seen
@@ -1178,6 +1177,18 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor
{
     return false;
   }
 
+  @VisibleForTesting
+  boolean shouldBufferUpdate(AddUpdateCommand cmd, boolean isReplayOrPeersync, UpdateLog.State
state) {
+    if (state == UpdateLog.State.APPLYING_BUFFERED
+        && !isReplayOrPeersync
+        && !cmd.isInPlaceUpdate()) {
+      // this a new update sent from the leader, it contains whole document therefore it
won't depend on other updates
+      return false;
+    }
+
+    return state != UpdateLog.State.ACTIVE && isReplayOrPeersync == false;
+  }
+
   /**
    * This method checks the update/transaction logs and index to find out if the update ("previous
update") that the current update
    * depends on (in the case that this current update is an in-place update) has already
been completed. If not,

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
new file mode 100644
index 0000000..8f56d68
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.UpdateLog;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solr/collection1/conf/solrconfig.xml","solr/collection1/conf/schema-minimal.xml");
+  }
+
+  @Test
+  public void testShouldBufferUpdate() {
+    SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new ModifiableSolrParams());
+    DistributedUpdateProcessor processor = new DistributedUpdateProcessor(
+        req, null, null, null);
+
+    AddUpdateCommand cmd = new AddUpdateCommand(req);
+    // applying buffer updates, isReplayOrPeerSync flag doesn't matter
+    assertFalse(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.APPLYING_BUFFERED));
+    assertFalse(processor.shouldBufferUpdate(cmd, true, UpdateLog.State.APPLYING_BUFFERED));
+
+    assertTrue(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.BUFFERING));
+    // this is not an buffer updates and it depend on other updates
+    cmd.prevVersion = 10;
+    assertTrue(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.APPLYING_BUFFERED));
+  }
+
+
+}


Mime
View raw message