lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject [1/2] lucene-solr:branch_6x: SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages
Date Tue, 09 May 2017 07:07:18 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/branch_6x 219442df9 -> 5c626dc9e


SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9cab9c0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9cab9c0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9cab9c0c

Branch: refs/heads/branch_6x
Commit: 9cab9c0cf2777a21a81386be2262e84da2bca751
Parents: 219442d
Author: Cao Manh Dat <datcm@apache.org>
Authored: Mon May 8 16:21:19 2017 +0700
Committer: Cao Manh Dat <datcm@apache.org>
Committed: Tue May 9 14:06:29 2017 +0700

----------------------------------------------------------------------
 solr/CHANGES.txt                                      |  2 ++
 .../core/src/java/org/apache/solr/cloud/Overseer.java |  3 ++-
 .../org/apache/solr/cloud/overseer/ZkStateWriter.java | 14 ++------------
 3 files changed, 6 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ec08c7f..e89ee4b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -154,6 +154,8 @@ Optimizations
 * SOLR-10548: JSON Facet API now uses hyper-log-log++ for determining the number of buckets
   when merging requests from a multi-shard distributed request. (yonik)
 
+* SOLR-10524: Explore in-memory partitioning for processing Overseer queue messages (Cao
Manh Dat, Noble Paul, shalin, Scott Blum)
+
 Bug Fixes
 ----------------------
 * SOLR-10281: ADMIN_PATHS is duplicated in two places and inconsistent. This can cause automatic

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index e27251e..1f28932 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -64,7 +64,8 @@ import static org.apache.solr.common.params.CommonParams.ID;
 public class Overseer implements Closeable {
   public static final String QUEUE_OPERATION = "operation";
 
-  public static final int STATE_UPDATE_DELAY = 1500;  // delay between cloud state updates
+  public static final int STATE_UPDATE_DELAY = 2000;  // delay between cloud state updates
+  public static final int STATE_UPDATE_BATCH_SIZE = 10000;
 
   public static final int NUM_RESPONSES_TO_STORE = 10000;
   

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9cab9c0c/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 23fb56c..48d2ecb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -166,17 +166,7 @@ public class ZkStateWriter {
    * @return true if a flush is required, false otherwise
    */
   protected boolean maybeFlushBefore(ZkWriteCommand cmd) {
-    if (lastUpdatedTime == 0) {
-      // first update, make sure we go through
-      return false;
-    }
-    if (cmd.collection == null) {
-      return false;
-    }
-    if (cmd.collection.getStateFormat() != lastStateFormat) {
-      return true;
-    }
-    return cmd.collection.getStateFormat() > 1 && !cmd.name.equals(lastCollectionName);
+    return cmd.collection.getStateFormat() != lastStateFormat;
   }
 
   /**
@@ -190,7 +180,7 @@ public class ZkStateWriter {
       return false;
     lastCollectionName = cmd.name;
     lastStateFormat = cmd.collection.getStateFormat();
-    return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL;
+    return System.nanoTime() - lastUpdatedTime > MAX_FLUSH_INTERVAL || updates.size()
> Overseer.STATE_UPDATE_BATCH_SIZE;
   }
 
   public boolean hasPendingUpdates() {


Mime
View raw message