lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sar...@apache.org
Subject [40/51] [abbrv] lucene-solr:apiv2: SOLR-7280: refactored to incorporate Mike's suggestions. Default thread count for cloud is limited to 8 now. In our internal teting 8 has given us the best stability during restarts
Date Thu, 21 Jul 2016 13:38:02 GMT
SOLR-7280: refactored to incorporate Mike's suggestions. Default thread count for cloud is
limited to 8 now. In our internal teting 8 has given us the best stability during restarts


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2d1496c8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2d1496c8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2d1496c8

Branch: refs/heads/apiv2
Commit: 2d1496c83d83bb6582af39af6cf272828d83c9e3
Parents: 5c4b717
Author: Noble Paul <noble@apache.org>
Authored: Tue Jul 19 18:21:23 2016 +0530
Committer: Noble Paul <noble@apache.org>
Committed: Tue Jul 19 18:21:23 2016 +0530

----------------------------------------------------------------------
 .../org/apache/solr/core/CoreContainer.java     | 29 ++++++++---------
 .../java/org/apache/solr/core/NodeConfig.java   | 12 ++++---
 .../org/apache/solr/core/CoreSorterTest.java    | 34 +++++++++-----------
 .../test/org/apache/solr/core/TestSolrXml.java  |  2 +-
 4 files changed, 36 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d1496c8/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index aa65f54..1bdf3e3 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -16,19 +16,6 @@
  */
 package org.apache.solr.core;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.util.Collections.EMPTY_MAP;
-import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
-import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
-import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.ZK_PATH;
-import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS;
-import static org.apache.solr.core.NodeConfig.NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD;
-import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
-
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Path;
@@ -46,6 +33,8 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 import org.apache.http.auth.AuthSchemeProvider;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.config.Lookup;
@@ -88,8 +77,16 @@ import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Collections.EMPTY_MAP;
+import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
+import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
+import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.ZK_PATH;
+import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
 
 
 /**
@@ -489,7 +486,7 @@ public class CoreContainer {
 
     // setup executor to load cores in parallel
     ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(
-        cfg.getCoreLoadThreadCount(isZooKeeperAware() ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD
: DEFAULT_CORE_LOAD_THREADS),
+        cfg.getCoreLoadThreadCount(isZooKeeperAware()),
         new DefaultSolrThreadFactory("coreLoadExecutor") );
     final List<Future<SolrCore>> futures = new ArrayList<>();
     try {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d1496c8/solr/core/src/java/org/apache/solr/core/NodeConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
index 3db453b..ea451a9 100644
--- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java
@@ -87,7 +87,7 @@ public class NodeConfig {
     this.solrProperties = solrProperties;
     this.backupRepositoryPlugins = backupRepositoryPlugins;
 
-    if (this.cloudConfig != null && this.getCoreLoadThreadCount(NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS)
< 2) {
+    if (this.cloudConfig != null && this.getCoreLoadThreadCount(false) < 2) {
       throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
           "SolrCloud requires a value of at least 2 for coreLoadThreads (configured value
= " + this.coreLoadThreads + ")");
     }
@@ -109,8 +109,10 @@ public class NodeConfig {
     return updateShardHandlerConfig;
   }
 
-  public int getCoreLoadThreadCount(int def) {
-    return coreLoadThreads == null ? def : coreLoadThreads;
+  public int getCoreLoadThreadCount(boolean zkAware) {
+    return coreLoadThreads == null ?
+        (zkAware ? NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : NodeConfigBuilder.DEFAULT_CORE_LOAD_THREADS)
+        : coreLoadThreads;
   }
 
   public String getSharedLibDirectory() {
@@ -196,8 +198,8 @@ public class NodeConfig {
     private final String nodeName;
 
     public static final int DEFAULT_CORE_LOAD_THREADS = 3;
-    //No:of core load threads in cloud mode is set to a default of 24
-    public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 24;
+    //No:of core load threads in cloud mode is set to a default of 8
+    public static final int DEFAULT_CORE_LOAD_THREADS_IN_CLOUD = 8;
 
     private static final int DEFAULT_TRANSIENT_CACHE_SIZE = Integer.MAX_VALUE;
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d1496c8/solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/CoreSorterTest.java b/solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
index 3e5af20..dda437a 100644
--- a/solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
+++ b/solr/core/src/test/org/apache/solr/core/CoreSorterTest.java
@@ -39,7 +39,7 @@ import org.apache.solr.util.MockCoreContainer;
 import static java.util.stream.Collectors.toList;
 import static org.apache.solr.core.CoreSorter.getShardName;
 import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
 
@@ -49,16 +49,16 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
 
   public void testComparator() {
     List<CountsForEachShard> l = new ArrayList<>();
-    //                           DN LIV  MY
-    l.add(new CountsForEachShard(1, 3, 1));
-    l.add(new CountsForEachShard(0, 3, 2));
-    l.add(new CountsForEachShard(0, 3, 3));
-    l.add(new CountsForEachShard(0, 3, 4));
-    l.add(new CountsForEachShard(1, 0, 2));
-    l.add(new CountsForEachShard(1, 0, 1));
-    l.add(new CountsForEachShard(2, 5, 1));
-    l.add(new CountsForEachShard(2, 4, 2));
-    l.add(new CountsForEachShard(2, 3, 3));
+    //                           DOWN LIVE  MY
+    l.add(new CountsForEachShard(1,     3,  1));
+    l.add(new CountsForEachShard(0,     3,  2));
+    l.add(new CountsForEachShard(0,     3,  3));
+    l.add(new CountsForEachShard(0,     3,  4));
+    l.add(new CountsForEachShard(1,     0,  2));
+    l.add(new CountsForEachShard(1,     0,  1));
+    l.add(new CountsForEachShard(2,     5,  1));
+    l.add(new CountsForEachShard(2,     4,  2));
+    l.add(new CountsForEachShard(2,     3,  3));
 
     List<CountsForEachShard> expected = Arrays.asList(
         new CountsForEachShard(0, 3, 2),
@@ -108,14 +108,10 @@ public class CoreSorterTest extends SolrTestCaseJ4 {
     ZkController mockZKC = createMock(ZkController.class);
     ClusterState mockClusterState = createMock(ClusterState.class);
     reset(mockCC, mockZKC, mockClusterState);
-    mockCC.isZooKeeperAware();
-    expectLastCall().andAnswer(() -> Boolean.TRUE).anyTimes();
-    mockCC.getZkController();
-    expectLastCall().andAnswer(() -> mockZKC).anyTimes();
-    mockClusterState.getLiveNodes();
-    expectLastCall().andAnswer(() -> liveNodes).anyTimes();
-    mockZKC.getClusterState();
-    expectLastCall().andAnswer(() -> mockClusterState).anyTimes();
+    expect(mockCC.isZooKeeperAware()).andReturn(Boolean.TRUE).anyTimes();
+    expect(mockCC.getZkController()).andReturn(mockZKC).anyTimes();
+    expect(mockClusterState.getLiveNodes()).andReturn(liveNodes).anyTimes();
+    expect(mockZKC.getClusterState()).andReturn(mockClusterState).anyTimes();
     replay(mockCC, mockZKC, mockClusterState);
     return mockCC;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d1496c8/solr/core/src/test/org/apache/solr/core/TestSolrXml.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrXml.java b/solr/core/src/test/org/apache/solr/core/TestSolrXml.java
index e005c9f..9224c4d 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrXml.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrXml.java
@@ -73,7 +73,7 @@ public class TestSolrXml extends SolrTestCaseJ4 {
     assertEquals("collection handler class", "testCollectionsHandler", cfg.getCollectionsHandlerClass());
     assertEquals("info handler class", "testInfoHandler", cfg.getInfoHandlerClass());
     assertEquals("config set handler class", "testConfigSetsHandler", cfg.getConfigSetsHandlerClass());
-    assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(0));
+    assertEquals("core load threads", 11, cfg.getCoreLoadThreadCount(false));
     assertThat("core root dir", cfg.getCoreRootDirectory().toString(), containsString("testCoreRootDirectory"));
     assertEquals("distrib conn timeout", 22, cfg.getUpdateShardHandlerConfig().getDistributedConnectionTimeout());
     assertEquals("distrib socket timeout", 33, cfg.getUpdateShardHandlerConfig().getDistributedSocketTimeout());


Mime
View raw message