lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject lucene-solr:jira/http2: SOLR-12640: Adding LBHttp2SolrClient and replace all usages in ShardHandler with the new one
Date Wed, 08 Aug 2018 11:00:14 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/jira/http2 d6d6fbdbe -> 51b2a1be3


SOLR-12640: Adding LBHttp2SolrClient and replace all usages in ShardHandler with the new one


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/51b2a1be
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/51b2a1be
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/51b2a1be

Branch: refs/heads/jira/http2
Commit: 51b2a1be39c010b83bc377c149a0f2c6d570f9df
Parents: d6d6fbd
Author: Cao Manh Dat <datcm@apache.org>
Authored: Wed Aug 8 17:59:57 2018 +0700
Committer: Cao Manh Dat <datcm@apache.org>
Committed: Wed Aug 8 17:59:57 2018 +0700

----------------------------------------------------------------------
 .../handler/component/HttpShardHandler.java     |  11 +-
 .../component/HttpShardHandlerFactory.java      |  65 +-
 .../java/org/apache/solr/update/PeerSync.java   |   6 +-
 .../apache/solr/update/UpdateShardHandler.java  |   3 +-
 .../solr/cloud/TestMiniSolrCloudClusterSSL.java |  11 +
 .../cloud/TestSolrCloudWithKerberosAlt.java     |   3 +
 .../TestSolrCloudWithSecureImpersonation.java   |   3 +
 .../component/TestHttpShardHandlerFactory.java  |   2 -
 .../PKIAuthenticationIntegrationTest.java       |   3 +
 .../TestSolrCloudWithHadoopAuthPlugin.java      |   3 +
 .../solr/client/solrj/impl/Http2SolrClient.java | 106 +--
 .../client/solrj/impl/LBHttp2SolrClient.java    | 696 +++++++++++++++++++
 .../client/solrj/TestLBHttp2SolrClient.java     | 333 +++++++++
 .../solr/BaseDistributedSearchTestCase.java     |   6 +-
 14 files changed, 1150 insertions(+), 101 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
index a548031..7986a1c 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
@@ -36,7 +36,9 @@ import org.apache.http.client.HttpClient;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
+import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
 import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.util.ClientUtils;
@@ -74,11 +76,11 @@ public class HttpShardHandler extends ShardHandler {
   private CompletionService<ShardResponse> completionService;
   private Set<Future<ShardResponse>> pending;
   private Map<String,List<String>> shardToURLs;
-  private HttpClient httpClient;
+  private Http2SolrClient httpClient;
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-  public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, HttpClient httpClient) {
+  public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, Http2SolrClient httpClient) {
     this.httpClient = httpClient;
     this.httpShardHandlerFactory = httpShardHandlerFactory;
     completionService = httpShardHandlerFactory.newCompletionService();
@@ -168,9 +170,8 @@ public class HttpShardHandler extends ShardHandler {
         if (urls.size() <= 1) {
           String url = urls.get(0);
           srsp.setShardAddress(url);
-          try (SolrClient client = new Builder(url).withHttpClient(httpClient).build()) {
-            ssr.nl = client.request(req);
-          }
+          req.setBasePath(url);
+          ssr.nl = httpClient.request(req);
         } else {
           LBHttpSolrClient.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
           ssr.nl = rsp.getResponse();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
index b0d3dd8..daef715 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
@@ -16,13 +16,27 @@
  */
 package org.apache.solr.handler.component;
 
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.commons.lang.StringUtils;
-import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
 import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
-import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
@@ -32,6 +46,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.IOUtils;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.URLUtil;
@@ -39,8 +54,8 @@ import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrInfoBean;
 import org.apache.solr.metrics.SolrMetricManager;
 import org.apache.solr.metrics.SolrMetricProducer;
-import org.apache.solr.update.UpdateShardHandlerConfig;
 import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.UpdateShardHandlerConfig;
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.apache.solr.util.stats.HttpClientMetricNameStrategy;
 import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
@@ -49,21 +64,6 @@ import org.apache.solr.util.stats.MetricUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-
 import static org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.KNOWN_METRIC_NAME_STRATEGIES;
 
 
@@ -89,9 +89,9 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
   );
 
   protected InstrumentedPoolingHttpClientConnectionManager clientConnectionManager;
-  protected CloseableHttpClient defaultClient;
+  protected Http2SolrClient defaultClient;
   protected InstrumentedHttpRequestExecutor httpRequestExecutor;
-  private LBHttpSolrClient loadbalancer;
+  private LBHttp2SolrClient loadbalancer;
   //default values:
   int soTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATESOTIMEOUT;
   int connectionTimeout = UpdateShardHandlerConfig.DEFAULT_DISTRIBUPDATECONNTIMEOUT;
@@ -150,7 +150,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
   /**
    * Get {@link ShardHandler} that uses custom http client.
    */
-  public ShardHandler getShardHandler(final HttpClient httpClient){
+  public ShardHandler getShardHandler(final Http2SolrClient httpClient){
     return new HttpShardHandler(this, httpClient);
   }
 
@@ -209,11 +209,11 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
         new DefaultSolrThreadFactory("httpShardExecutor")
     );
 
-    ModifiableSolrParams clientParams = getClientParams();
+    //TODO no commit, replace all metrics
     httpRequestExecutor = new InstrumentedHttpRequestExecutor(this.metricNameStrategy);
     clientConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
-    this.defaultClient = HttpClientUtil.createClient(clientParams, clientConnectionManager, false, httpRequestExecutor);
-    this.loadbalancer = createLoadbalancer(defaultClient);
+    this.defaultClient = new Http2SolrClient.Builder().connectionTimeout(connectionTimeout).idleTimeout(soTimeout).build();
+    this.loadbalancer = new LBHttp2SolrClient(defaultClient);
   }
 
   protected ModifiableSolrParams getClientParams() {
@@ -223,19 +223,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
     return clientParams;
   }
 
-  protected ExecutorService getThreadPoolExecutor(){
-    return this.commExecutor;
-  }
-
-  protected LBHttpSolrClient createLoadbalancer(HttpClient httpClient){
-    LBHttpSolrClient client = new Builder()
-        .withHttpClient(httpClient)
-        .withConnectionTimeout(connectionTimeout)
-        .withSocketTimeout(soTimeout)
-        .build();
-    return client;
-  }
-
   protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue, StringBuilder sb) {
     T toReturn = defaultValue;
     if (initArgs != null) {
@@ -258,7 +245,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
         }
       } finally { 
         if (defaultClient != null) {
-          HttpClientUtil.close(defaultClient);
+          IOUtils.closeQuietly(defaultClient);
         }
         if (clientConnectionManager != null)  {
           clientConnectionManager.close();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/java/org/apache/solr/update/PeerSync.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index 6c3c578..eb87916 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -36,6 +36,7 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.lucene.util.BytesRef;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
@@ -85,7 +86,7 @@ public class PeerSync implements SolrMetricProducer {
 
   private final boolean cantReachIsSuccess;
   private final boolean doFingerprint;
-  private final HttpClient client;
+  private final Http2SolrClient client;
   private final boolean onlyIfActive;
   private SolrCore core;
   private Updater updater;
@@ -117,7 +118,8 @@ public class PeerSync implements SolrMetricProducer {
     this.nUpdates = nUpdates;
     this.cantReachIsSuccess = cantReachIsSuccess;
     this.doFingerprint = doFingerprint && !("true".equals(System.getProperty("solr.disableFingerprint")));
-    this.client = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
+    //TODO nocommit, replace it with default client
+    this.client = core.getCoreContainer().getUpdateShardHandler().getUpdateOnlyHttpClient();
     this.onlyIfActive = onlyIfActive;
     
     uhandler = core.getUpdateHandler();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index 47926bc..19b09c1 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -111,7 +111,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
 
 
     httpRequestExecutor = new InstrumentedHttpRequestExecutor(metricNameStrategy);
-    Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder("");
+    Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder();
     if (cfg != null) {
       updateOnlyClientBuilder.connectionTimeout(cfg.getDistributedConnectionTimeout())
           .idleTimeout(cfg.getDistributedSocketTimeout());
@@ -147,6 +147,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
   public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) {
     registry = manager.registry(registryName);
     String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
+    //TODO nocommit, replace all the metrics
     updateOnlyConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
     defaultConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
     updateExecutor = MetricUtils.instrumentedExecutorService(updateExecutor, this, registry,

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
index eeb7be7..d470bb6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
@@ -41,6 +41,7 @@ import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettyConfig;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -88,11 +89,13 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
     // undo the randomization of our super class
     log.info("NOTE: This Test ignores the randomized SSL & clientAuth settings selected by base class");
     HttpClientUtil.resetHttpClientBuilder(); // also resets SchemaRegistryProvider
+    Http2SolrClient.resetSslContextFactory();
     System.clearProperty(ZkStateReader.URL_SCHEME);
   }
   @After
   public void after() {
     HttpClientUtil.resetHttpClientBuilder(); // also resets SchemaRegistryProvider
+    Http2SolrClient.resetSslContextFactory();
     System.clearProperty(ZkStateReader.URL_SCHEME);
     SSLContext.setDefault(DEFAULT_SSL_CONTEXT);
   }
@@ -100,6 +103,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
   public void testNoSsl() throws Exception {
     final SSLTestConfig sslConfig = new SSLTestConfig(false, false);
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "http");
     checkClusterWithNodeReplacement(sslConfig);
   }
@@ -110,6 +114,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
     // options.
     final SSLTestConfig sslConfig = new SSLTestConfig(false, true);
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "http");
     checkClusterWithNodeReplacement(sslConfig);
   }
@@ -117,6 +122,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
   public void testSslAndNoClientAuth() throws Exception {
     final SSLTestConfig sslConfig = new SSLTestConfig(true, false);
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "https");
     checkClusterWithNodeReplacement(sslConfig);
   }
@@ -127,6 +133,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
     final SSLTestConfig sslConfig = new SSLTestConfig(true, true);
 
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "https");
     checkClusterWithNodeReplacement(sslConfig);
   }
@@ -135,6 +142,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
   public void testSslWithCheckPeerName() throws Exception {
     final SSLTestConfig sslConfig = new SSLTestConfig(true, false, true);
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "https");
     checkClusterWithNodeReplacement(sslConfig);
   }
@@ -164,6 +172,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
       System.setProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME,
                          Boolean.toString(sslConfig.getCheckPeerName()));
       HttpClientUtil.resetHttpClientBuilder();
+      Http2SolrClient.resetSslContextFactory();
       
       // recheck that we can communicate with all the jetty instances in our cluster
       checkClusterJettys(cluster, sslConfig);
@@ -178,6 +187,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
     // certs with a bogus hostname/ip and clients shouldn't care...
     final SSLTestConfig sslConfig = new SSLTestConfig(true, false, false);
     HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
+    Http2SolrClient.setSslContextFactory(sslConfig.buildSslContextFactory());
     System.setProperty(ZkStateReader.URL_SCHEME, "https");
     final JettyConfig config = JettyConfig.builder().withSSLConfig(sslConfig).build();
     final MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), config);
@@ -188,6 +198,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
       // our existing certificate, but *does* care about validating the peer name
       System.setProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME, "true");
       HttpClientUtil.resetHttpClientBuilder();
+      Http2SolrClient.resetSslContextFactory();
 
       // and validate we get failures when trying to talk to our cluster...
       final List<JettySolrRunner> jettys = cluster.getJettySolrRunners();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
index 4317736..d231b31 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
@@ -31,6 +31,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.util.BadZookeeperThreadsFilter;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,6 +47,8 @@ import org.slf4j.LoggerFactory;
 })
 
 @LuceneTestCase.Slow
+//TODO nocommit
+@Ignore
 public class TestSolrCloudWithKerberosAlt extends SolrCloudTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
index 184cd90..f392583 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
@@ -45,6 +45,7 @@ import org.apache.solr.servlet.SolrRequestParsers;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.apache.solr.security.HttpParamDelegationTokenPlugin.REMOTE_ADDRESS_PARAM;
@@ -336,6 +337,8 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
   }
 
   @Test
+  @Ignore
+  //TODO nocommit
   public void testForwarding() throws Exception {
     String collectionName = "forwardingCollection";
     miniCluster.uploadConfigSet(TEST_PATH().resolve("collection1/conf"), "conf1");

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
index 523e31d..01aff97 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
@@ -29,8 +29,6 @@ import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.core.CoreContainer;
-import org.apache.solr.handler.component.HttpShardHandlerFactory;
-import org.apache.solr.handler.component.ShardHandlerFactory;
 
 import org.junit.BeforeClass;
 import org.junit.AfterClass;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
index 1f85f3a..61dd315 100644
--- a/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/security/PKIAuthenticationIntegrationTest.java
@@ -31,6 +31,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.Utils;
 import org.junit.After;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -39,6 +40,8 @@ import static java.util.Collections.singletonMap;
 import static org.apache.solr.common.util.Utils.makeMap;
 import static org.apache.solr.security.TestAuthorizationFramework.verifySecurityStatus;
 
+@Ignore
+//TODO nocommit
 public class PKIAuthenticationIntegrationTest extends SolrCloudTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
index 118dc47..ff5baf9 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java
@@ -31,9 +31,12 @@ import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.SolrInputDocument;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 // commented 20-July-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
+@Ignore
+//TODO nocommit
 public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudTestCase {
   protected static final int NUM_SERVERS = 1;
   protected static final int NUM_SHARDS = 1;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index ebcde3e..6241188 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -21,7 +21,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
-import java.lang.invoke.MethodHandles;
 import java.net.ConnectException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -82,8 +81,6 @@ import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
 import org.eclipse.jetty.util.Fields;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import static org.apache.solr.common.util.Utils.getObjectByPath;
 
@@ -92,42 +89,7 @@ import static org.apache.solr.common.util.Utils.getObjectByPath;
  * @lucene.experimental
  */
 public class Http2SolrClient extends SolrClient {
-  private static volatile SslContextFactory defaultSslContextFactory;
-
-  static {
-    defaultSslContextFactory = new SslContextFactory(false);
-
-    if (null != System.getProperty("javax.net.ssl.keyStore")) {
-      defaultSslContextFactory.setKeyStorePath
-          (System.getProperty("javax.net.ssl.keyStore"));
-    }
-    if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
-      defaultSslContextFactory.setKeyStorePassword
-          (System.getProperty("javax.net.ssl.keyStorePassword"));
-    }
-    if (null != System.getProperty("javax.net.ssl.trustStore")) {
-      defaultSslContextFactory.setTrustStorePath
-          (System.getProperty("javax.net.ssl.trustStore"));
-    }
-    if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
-      defaultSslContextFactory.setTrustStorePassword
-          (System.getProperty("javax.net.ssl.trustStorePassword"));
-    }
-
-    String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
-    boolean sslCheckPeerName = true;
-    if (checkPeerNameStr == null && "false".equalsIgnoreCase(checkPeerNameStr)) {
-      sslCheckPeerName = false;
-    }
-
-    if (System.getProperty("tests.jettySsl.clientAuth") != null) {
-      sslCheckPeerName = sslCheckPeerName || Boolean.getBoolean("tests.jettySsl.clientAuth");
-    }
-    defaultSslContextFactory.setNeedClientAuth(sslCheckPeerName);
-
-  }
-
-  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private static volatile SslContextFactory defaultSslContextFactory = getDefaultSslContextFactory();
   private static final int MAX_OUTSTANDING_REQUESTS = 1000;
   private static final String AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
   private static final String UTF_8 = StandardCharsets.UTF_8.name();
@@ -156,12 +118,8 @@ public class Http2SolrClient extends SolrClient {
     }
   };
 
-  private Request.BeginListener requestBeginListener = new Request.BeginListener() {
-
-    @Override
-    public void onBegin(Request arg0) {
+  private Request.BeginListener beginListener = req -> {
 
-    }
   };
 
   private Response.CompleteListener requestCompleteListener = new Response.CompleteListener() {
@@ -177,7 +135,6 @@ public class Http2SolrClient extends SolrClient {
    * The URL of the Solr server.
    */
   private String serverBaseUrl;
-
   private boolean closeClient;
 
   protected Http2SolrClient(String serverBaseUrl, Builder builder) {
@@ -202,6 +159,7 @@ public class Http2SolrClient extends SolrClient {
     } else {
       httpClient = builder.httpClient;
     }
+    if (builder.beginListener != null) this.beginListener = builder.beginListener;
     if (!httpClient.isStarted()) {
       try {
         httpClient.start();
@@ -239,7 +197,6 @@ public class Http2SolrClient extends SolrClient {
 
     if (builder.idleTimeout != null) httpClient.setIdleTimeout(builder.idleTimeout);
     if (builder.connectionTimeout != null) httpClient.setConnectTimeout(builder.connectionTimeout);
-
     return httpClient;
   }
 
@@ -248,7 +205,7 @@ public class Http2SolrClient extends SolrClient {
     return httpClient;
   }
 
-  public void close() throws IOException {
+  public void close() {
     // we wait for async requests, so far devs don't want to give sugar for this
     phaser.arriveAndAwaitAdvance();
     phaser.arriveAndDeregister();
@@ -286,7 +243,7 @@ public class Http2SolrClient extends SolrClient {
     Request req = makeRequest(solrRequest, collection);
     try {
       if (onComplete != null) {
-        req.onRequestBegin(requestBeginListener).onRequestQueued(requestQueuedListener)
+        req.onRequestBegin(beginListener).onRequestQueued(requestQueuedListener)
             .onComplete(requestCompleteListener).send(new BufferingResponseListener() {
 
           @Override
@@ -316,13 +273,13 @@ public class Http2SolrClient extends SolrClient {
         Http2ClientResponse arsp = new Http2ClientResponse();
         if (returnStream) {
           InputStreamResponseListener listener = new InputStreamResponseListener();
-          req.send(listener);
+          req.onRequestBegin(beginListener).send(listener);
           // Wait for the response headers to arrive
           listener.get(idleTimeout, TimeUnit.SECONDS);
           // TODO: process response
           arsp.stream = listener.getInputStream();
         } else {
-          ContentResponse response = req.send();
+          ContentResponse response = req.onRequestBegin(beginListener).send();
           ByteArrayInputStream is = new ByteArrayInputStream(response.getContent());
           arsp.response = processErrorsAndResponse(response, parser,
               is, response.getEncoding(), isV2ApiRequest(solrRequest));
@@ -354,6 +311,8 @@ public class Http2SolrClient extends SolrClient {
 
   private Request makeRequest(SolrRequest solrRequest, String collection)
       throws SolrServerException, IOException {
+    if (solrRequest.getBasePath() == null && serverBaseUrl == null)
+      throw new IllegalArgumentException("Destination node is not provided!");
 
     if (solrRequest instanceof V2RequestSupport) {
       solrRequest = ((V2RequestSupport) solrRequest).getV2Request();
@@ -658,6 +617,11 @@ public class Http2SolrClient extends SolrClient {
     private Integer connectionTimeout;
     private boolean useHttp1_1 = false;
     protected String baseSolrUrl;
+    private Request.BeginListener beginListener = request -> {};
+
+    public Builder() {
+      this.sslContextFactory = defaultSslContextFactory;
+    }
 
     public Builder(String baseSolrUrl) {
       this.baseSolrUrl = baseSolrUrl;
@@ -692,6 +656,11 @@ public class Http2SolrClient extends SolrClient {
       this.connectionTimeout = connectionTimeOut;
       return this;
     }
+
+    public Builder withListener(Request.BeginListener beginListener) {
+      this.beginListener = beginListener;
+      return this;
+    }
   }
 
   /**
@@ -793,6 +762,41 @@ public class Http2SolrClient extends SolrClient {
 
   // public for testing, only used by tests
   public static void resetSslContextFactory() {
-    Http2SolrClient.defaultSslContextFactory = null;
+    Http2SolrClient.defaultSslContextFactory = getDefaultSslContextFactory();
+  }
+
+  private static SslContextFactory getDefaultSslContextFactory() {
+    SslContextFactory sslContextFactory = new SslContextFactory(false);
+
+    if (null != System.getProperty("javax.net.ssl.keyStore")) {
+      sslContextFactory.setKeyStorePath
+          (System.getProperty("javax.net.ssl.keyStore"));
+    }
+    if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
+      sslContextFactory.setKeyStorePassword
+          (System.getProperty("javax.net.ssl.keyStorePassword"));
+    }
+    if (null != System.getProperty("javax.net.ssl.trustStore")) {
+      sslContextFactory.setTrustStorePath
+          (System.getProperty("javax.net.ssl.trustStore"));
+    }
+    if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
+      sslContextFactory.setTrustStorePassword
+          (System.getProperty("javax.net.ssl.trustStorePassword"));
+    }
+
+    String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
+    boolean sslCheckPeerName = true;
+    if (checkPeerNameStr == null && "false".equalsIgnoreCase(checkPeerNameStr)) {
+      sslCheckPeerName = false;
+    }
+
+    if (System.getProperty("tests.jettySsl.clientAuth") != null) {
+      sslCheckPeerName = sslCheckPeerName || Boolean.getBoolean("tests.jettySsl.clientAuth");
+    }
+
+    sslContextFactory.setNeedClientAuth(sslCheckPeerName);
+    return sslContextFactory;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
new file mode 100644
index 0000000..b1860f1
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java
@@ -0,0 +1,696 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteExecutionException;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Req;
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Rsp;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.slf4j.MDC;
+
+import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
+
+/**
+ * LBHttp2SolrClient or "LoadBalanced LBHttp2SolrClient" is a load balancing wrapper around
+ * {@link Http2SolrClient}. This is useful when you
+ * have multiple Solr servers and the requests need to be Load Balanced among them.
+ *
+ * Do <b>NOT</b> use this class for indexing in master/slave scenarios since documents must be sent to the
+ * correct master; no inter-node routing is done.
+ *
+ * In SolrCloud (leader/replica) scenarios, it is usually better to use
+ * {@link CloudSolrClient}, but this class may be used
+ * for updates because the server will forward them to the appropriate leader.
+ *
+ * <p>
+ * It offers automatic failover when a server goes down and it detects when the server comes back up.
+ * <p>
+ * Load balancing is done using a simple round-robin on the list of servers.
+ * <p>
+ * If a request to a server fails by an IOException due to a connection timeout or read timeout then the host is taken
+ * off the list of live servers and moved to a 'dead server list' and the request is resent to the next live server.
+ * This process is continued till it tries all the live servers. If at least one server is alive, the request succeeds,
+ * and if not it fails.
+ * <blockquote><pre>
+ * SolrClient lbHttp2SolrClient = new LBHttp2SolrClient(http2SolrClient, "http://host1:8080/solr/", "http://host2:8080/solr", "http://host2:8080/solr");
+ * </pre></blockquote>
+ * This detects if a dead server comes alive automatically. The check is done in fixed intervals in a dedicated thread.
+ * This interval can be set using {@link #setAliveCheckInterval} , the default is set to one minute.
+ * <p>
+ * <b>When to use this?</b><br> This can be used as a software load balancer when you do not wish to setup an external
+ * load balancer. Alternatives to this code are to use
+ * a dedicated hardware load balancer or using Apache httpd with mod_proxy_balancer as a load balancer. See <a
+ * href="http://en.wikipedia.org/wiki/Load_balancing_(computing)">Load balancing on Wikipedia</a>
+ *
+ * @since solr 1.4
+ */
+public class LBHttp2SolrClient extends SolrClient {
+  private static Set<Integer> RETRY_CODES = new HashSet<>(4);
+
+  static {
+    RETRY_CODES.add(404);
+    RETRY_CODES.add(403);
+    RETRY_CODES.add(503);
+    RETRY_CODES.add(500);
+  }
+
+  // keys to the maps are currently of the form "http://localhost:8983/solr"
+  private final Map<String, ServerWrapper> aliveServers = new LinkedHashMap<>();
+  // access to aliveServers should be synchronized on itself
+
+  protected final Map<String, ServerWrapper> zombieServers = new ConcurrentHashMap<>();
+
+  // changes to aliveServers are reflected in this array, no need to synchronize
+  private volatile ServerWrapper[] aliveServerList = new ServerWrapper[0];
+
+
+  private ScheduledExecutorService aliveCheckExecutor;
+
+  private final Http2SolrClient httpClient;
+  private final AtomicInteger counter = new AtomicInteger(-1);
+
+  private static final SolrQuery solrQuery = new SolrQuery("*:*");
+  private volatile ResponseParser parser;
+  private volatile RequestWriter requestWriter;
+
+  private Set<String> queryParams = new HashSet<>();
+  private Integer connectionTimeout;
+
+  private Integer soTimeout;
+
+  static {
+    solrQuery.setRows(0);
+    /**
+     * Default sort (if we don't supply a sort) is by score and since
+     * we request 0 rows any sorting and scoring is not necessary.
+     * SolrQuery.DOCID schema-independently specifies a non-scoring sort.
+     * <code>_docid_ asc</code> sort is efficient,
+     * <code>_docid_ desc</code> sort is not, so choose ascending DOCID sort.
+     */
+    solrQuery.setSort(SolrQuery.DOCID, SolrQuery.ORDER.asc);
+    // not a top-level request, we are interested only in the server being sent to i.e. it need not distribute our request to further servers
+    solrQuery.setDistrib(false);
+  }
+
+  protected static class ServerWrapper {
+
+    final String baseUrl;
+
+    // "standard" servers are used by default.  They normally live in the alive list
+    // and move to the zombie list when unavailable.  When they become available again,
+    // they move back to the alive list.
+    boolean standard = true;
+
+    int failedPings = 0;
+
+    public ServerWrapper(String baseUrl) {
+      this.baseUrl = baseUrl;
+    }
+
+    @Override
+    public String toString() {
+      return baseUrl;
+    }
+
+    @Override
+    public int hashCode() {
+      return baseUrl.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) return true;
+      if (!(obj instanceof ServerWrapper)) return false;
+      return baseUrl.equals(((ServerWrapper)obj).baseUrl);
+    }
+  }
+
+  public LBHttp2SolrClient(Http2SolrClient httpClient, String... baseSolrUrls) {
+    this.httpClient = httpClient;
+    if (baseSolrUrls != null) {
+      for (String s : baseSolrUrls) {
+        ServerWrapper wrapper = new ServerWrapper(s);
+        aliveServers.put(s, wrapper);
+      }
+    }
+    updateAliveList();
+  }
+
+  public Set<String> getQueryParams() {
+    return queryParams;
+  }
+
+  /**
+   * Expert Method.
+   * @param queryParams set of param keys to only send via the query string
+   */
+  public void setQueryParams(Set<String> queryParams) {
+    this.queryParams = queryParams;
+  }
+  public void addQueryParams(String queryOnlyParam) {
+    this.queryParams.add(queryOnlyParam) ;
+  }
+
+  public static String normalize(String server) {
+    if (server.endsWith("/"))
+      server = server.substring(0, server.length() - 1);
+    return server;
+  }
+
+  /**
+   * Tries to query a live server from the list provided in Req. Servers in the dead pool are skipped.
+   * If a request fails due to an IOException, the server is moved to the dead pool for a certain period of
+   * time, or until a test request on that server succeeds.
+   *
+   * Servers are queried in the exact order given (except servers currently in the dead pool are skipped).
+   * If no live servers from the provided list remain to be tried, a number of previously skipped dead servers will be tried.
+   * Req.getNumDeadServersToTry() controls how many dead servers will be tried.
+   *
+   * If no live servers are found a SolrServerException is thrown.
+   *
+   * @param req contains both the request as well as the list of servers to query
+   *
+   * @return the result of the request
+   *
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public Rsp request(Req req) throws SolrServerException, IOException {
+    Rsp rsp = new Rsp();
+    Exception ex = null;
+    boolean isNonRetryable = req.request instanceof IsUpdateRequest || ADMIN_PATHS.contains(req.request.getPath());
+    List<ServerWrapper> skipped = null;
+
+    final Integer numServersToTry = req.getNumServersToTry();
+    int numServersTried = 0;
+
+    boolean timeAllowedExceeded = false;
+    long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
+    long timeOutTime = System.nanoTime() + timeAllowedNano;
+    for (String serverStr : req.getServers()) {
+      if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
+        break;
+      }
+
+      serverStr = normalize(serverStr);
+      // if the server is currently a zombie, just skip to the next one
+      ServerWrapper wrapper = zombieServers.get(serverStr);
+      if (wrapper != null) {
+        // System.out.println("ZOMBIE SERVER QUERIED: " + serverStr);
+        final int numDeadServersToTry = req.getNumDeadServersToTry();
+        if (numDeadServersToTry > 0) {
+          if (skipped == null) {
+            skipped = new ArrayList<>(numDeadServersToTry);
+            skipped.add(wrapper);
+          }
+          else if (skipped.size() < numDeadServersToTry) {
+            skipped.add(wrapper);
+          }
+        }
+        continue;
+      }
+      try {
+        MDC.put("LBHttp2SolrClient.url", serverStr);
+
+        if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+          break;
+        }
+
+        ++numServersTried;
+        ex = doRequest(serverStr, req, rsp, isNonRetryable, false);
+        if (ex == null) {
+          return rsp; // SUCCESS
+        }
+      } finally {
+        MDC.remove("LBHttp2SolrClient.url");
+      }
+    }
+
+    // try the servers we previously skipped
+    if (skipped != null) {
+      for (ServerWrapper wrapper : skipped) {
+        if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
+          break;
+        }
+
+        if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+          break;
+        }
+
+        try {
+          MDC.put("LBHttp2SolrClient.url", wrapper.baseUrl);
+          ++numServersTried;
+          ex = doRequest(wrapper.baseUrl, req, rsp, isNonRetryable, true);
+          if (ex == null) {
+            return rsp; // SUCCESS
+          }
+        } finally {
+          MDC.remove("LBHttp2SolrClient.url");
+        }
+      }
+    }
+
+
+    final String solrServerExceptionMessage;
+    if (timeAllowedExceeded) {
+      solrServerExceptionMessage = "Time allowed to handle this request exceeded";
+    } else {
+      if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+        solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+            + " numServersTried="+numServersTried
+            + " numServersToTry="+numServersToTry.intValue();
+      } else {
+        solrServerExceptionMessage = "No live SolrServers available to handle this request";
+      }
+    }
+    if (ex == null) {
+      throw new SolrServerException(solrServerExceptionMessage);
+    } else {
+      throw new SolrServerException(solrServerExceptionMessage+":" + zombieServers.keySet(), ex);
+    }
+
+  }
+
+  protected Exception addZombie(String serverStr, Exception e) {
+
+    ServerWrapper wrapper;
+
+    wrapper = new ServerWrapper(serverStr);
+    wrapper.standard = false;
+    zombieServers.put(serverStr, wrapper);
+    startAliveCheckExecutor();
+    return e;
+  }
+
+  protected Exception doRequest(String serverStr, Req req, Rsp rsp, boolean isNonRetryable,
+      boolean isZombie) throws SolrServerException, IOException {
+    Exception ex = null;
+    try {
+      rsp.server = serverStr;
+      req.getRequest().setBasePath(serverStr);
+      rsp.rsp = httpClient.request(req.getRequest(), null);
+      if (isZombie) {
+        zombieServers.remove(serverStr);
+      }
+    } catch (RemoteExecutionException e){
+      throw e;
+    } catch(SolrException e) {
+      // we retry on 404 or 403 or 503 or 500
+      // unless it's an update - then we only retry on connect exception
+      if (!isNonRetryable && RETRY_CODES.contains(e.code())) {
+        ex = (!isZombie) ? addZombie(serverStr, e) : e;
+      } else {
+        // Server is alive but the request was likely malformed or invalid
+        if (isZombie) {
+          zombieServers.remove(serverStr);
+        }
+        throw e;
+      }
+    } catch (SocketException e) {
+      if (!isNonRetryable || e instanceof ConnectException) {
+        ex = (!isZombie) ? addZombie(serverStr, e) : e;
+      } else {
+        throw e;
+      }
+    } catch (SocketTimeoutException e) {
+      if (!isNonRetryable) {
+        ex = (!isZombie) ? addZombie(serverStr, e) : e;
+      } else {
+        throw e;
+      }
+    } catch (SolrServerException e) {
+      Throwable rootCause = e.getRootCause();
+      if (!isNonRetryable && rootCause instanceof IOException) {
+        ex = (!isZombie) ? addZombie(serverStr, e) : e;
+      } else if (isNonRetryable && rootCause instanceof ConnectException) {
+        ex = (!isZombie) ? addZombie(serverStr, e) : e;
+      } else {
+        throw e;
+      }
+    } catch (Exception e) {
+      throw new SolrServerException(e);
+    }
+
+    return ex;
+  }
+
+  private void updateAliveList() {
+    synchronized (aliveServers) {
+      aliveServerList = aliveServers.values().toArray(new ServerWrapper[aliveServers.size()]);
+    }
+  }
+
+  private ServerWrapper removeFromAlive(String key) {
+    synchronized (aliveServers) {
+      ServerWrapper wrapper = aliveServers.remove(key);
+      if (wrapper != null)
+        updateAliveList();
+      return wrapper;
+    }
+  }
+
+  private void addToAlive(ServerWrapper wrapper) {
+    synchronized (aliveServers) {
+      ServerWrapper prev = aliveServers.put(wrapper.baseUrl, wrapper);
+      // TODO: warn if there was a previous entry?
+      updateAliveList();
+    }
+  }
+
+  public void addSolrServer(String server) throws MalformedURLException {
+    addToAlive(new ServerWrapper(server));
+  }
+
+  public String removeSolrServer(String server) {
+    try {
+      server = new URL(server).toExternalForm();
+    } catch (MalformedURLException e) {
+      throw new RuntimeException(e);
+    }
+    if (server.endsWith("/")) {
+      server = server.substring(0, server.length() - 1);
+    }
+
+    // there is a small race condition here - if the server is in the process of being moved between
+    // lists, we could fail to remove it.
+    removeFromAlive(server);
+    zombieServers.remove(server);
+    return null;
+  }
+
+
+  @Override
+  public void close() {
+    if (aliveCheckExecutor != null) {
+      aliveCheckExecutor.shutdownNow();
+    }
+  }
+
+  /**
+   * Tries to query a live server. A SolrServerException is thrown if all servers are dead.
+   * If the request failed due to IOException then the live server is moved to dead pool and the request is
+   * retried on another live server.  After live servers are exhausted, any servers previously marked as dead
+   * will be tried before failing the request.
+   *
+   * @param request the SolrRequest.
+   *
+   * @return response
+   *
+   * @throws IOException If there is a low-level I/O error.
+   */
+  @Override
+  public NamedList<Object> request(final SolrRequest request, String collection)
+          throws SolrServerException, IOException {
+    return request(request, collection, null);
+  }
+
+  public NamedList<Object> request(final SolrRequest request, String collection,
+      final Integer numServersToTry) throws SolrServerException, IOException {
+    Exception ex = null;
+    ServerWrapper[] serverList = aliveServerList;
+
+    final int maxTries = (numServersToTry == null ? serverList.length : numServersToTry.intValue());
+    int numServersTried = 0;
+    Map<String,ServerWrapper> justFailed = null;
+
+    boolean timeAllowedExceeded = false;
+    long timeAllowedNano = getTimeAllowedInNanos(request);
+    long timeOutTime = System.nanoTime() + timeAllowedNano;
+    for (int attempts=0; attempts<maxTries; attempts++) {
+      if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
+        break;
+      }
+
+      int count = counter.incrementAndGet() & Integer.MAX_VALUE;
+      ServerWrapper wrapper = serverList[count % serverList.length];
+
+      try {
+        ++numServersTried;
+        request.setBasePath(wrapper.baseUrl);
+        return httpClient.request(request, collection);
+      } catch (SolrException e) {
+        // Server is alive but the request was malformed or invalid
+        throw e;
+      } catch (SolrServerException e) {
+        if (e.getRootCause() instanceof IOException) {
+          ex = e;
+          moveAliveToDead(wrapper);
+          if (justFailed == null) justFailed = new HashMap<>();
+          justFailed.put(wrapper.baseUrl, wrapper);
+        } else {
+          throw e;
+        }
+      } catch (Exception e) {
+        throw new SolrServerException(e);
+      }
+    }
+
+    // try other standard servers that we didn't try just now
+    for (ServerWrapper wrapper : zombieServers.values()) {
+      if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
+        break;
+      }
+
+      if (wrapper.standard==false || justFailed!=null && justFailed.containsKey(wrapper.baseUrl)) continue;
+      try {
+        ++numServersTried;
+        request.setBasePath(wrapper.baseUrl);
+        NamedList<Object> rsp = httpClient.request(request, collection);
+        // remove from zombie list *before* adding to alive to avoid a race that could lose a server
+        zombieServers.remove(wrapper.baseUrl);
+        addToAlive(wrapper);
+        return rsp;
+      } catch (SolrException e) {
+        // Server is alive but the request was malformed or invalid
+        throw e;
+      } catch (SolrServerException e) {
+        if (e.getRootCause() instanceof IOException) {
+          ex = e;
+          // still dead
+        } else {
+          throw e;
+        }
+      } catch (Exception e) {
+        throw new SolrServerException(e);
+      }
+    }
+
+
+    final String solrServerExceptionMessage;
+    if (timeAllowedExceeded) {
+      solrServerExceptionMessage = "Time allowed to handle this request exceeded";
+    } else {
+      if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
+        solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+            + " numServersTried="+numServersTried
+            + " numServersToTry="+numServersToTry.intValue();
+      } else {
+        solrServerExceptionMessage = "No live SolrServers available to handle this request";
+      }
+    }
+    if (ex == null) {
+      throw new SolrServerException(solrServerExceptionMessage);
+    } else {
+      throw new SolrServerException(solrServerExceptionMessage, ex);
+    }
+  }
+
+  /**
+   * @return time allowed in nanos, returns -1 if no time_allowed is specified.
+   */
+  private long getTimeAllowedInNanos(final SolrRequest req) {
+    SolrParams reqParams = req.getParams();
+    return reqParams == null ? -1 :
+      TimeUnit.NANOSECONDS.convert(reqParams.getInt(CommonParams.TIME_ALLOWED, -1), TimeUnit.MILLISECONDS);
+  }
+
+  private boolean isTimeExceeded(long timeAllowedNano, long timeOutTime) {
+    return timeAllowedNano > 0 && System.nanoTime() > timeOutTime;
+  }
+
+  /**
+   * Takes up one dead server and check for aliveness. The check is done in a roundrobin. Each server is checked for
+   * aliveness once in 'x' millis where x is decided by the setAliveCheckinterval() or it is defaulted to 1 minute
+   *
+   * @param zombieServer a server in the dead pool
+   */
+  private void checkAZombieServer(ServerWrapper zombieServer) {
+    try {
+      QueryRequest queryRequest = new QueryRequest(solrQuery);
+      queryRequest.setBasePath(zombieServer.baseUrl);
+      QueryResponse resp = queryRequest.process(httpClient);
+      if (resp.getStatus() == 0) {
+        // server has come back up.
+        // make sure to remove from zombies before adding to alive to avoid a race condition
+        // where another thread could mark it down, move it back to zombie, and then we delete
+        // from zombie and lose it forever.
+        ServerWrapper wrapper = zombieServers.remove(zombieServer.baseUrl);
+        if (wrapper != null) {
+          wrapper.failedPings = 0;
+          if (wrapper.standard) {
+            addToAlive(wrapper);
+          }
+        } else {
+          // something else already moved the server from zombie to alive
+        }
+      }
+    } catch (Exception e) {
+      //Expected. The server is still down.
+      zombieServer.failedPings++;
+
+      // If the server doesn't belong in the standard set belonging to this load balancer
+      // then simply drop it after a certain number of failed pings.
+      if (!zombieServer.standard && zombieServer.failedPings >= NONSTANDARD_PING_LIMIT) {
+        zombieServers.remove(zombieServer.baseUrl);
+      }
+    }
+  }
+
+  private void moveAliveToDead(ServerWrapper wrapper) {
+    wrapper = removeFromAlive(wrapper.baseUrl);
+    if (wrapper == null)
+      return;  // another thread already detected the failure and removed it
+    zombieServers.put(wrapper.baseUrl, wrapper);
+    startAliveCheckExecutor();
+  }
+
+  private int interval = CHECK_INTERVAL;
+
+  /**
+   * LBHttp2SolrClient keeps pinging the dead servers at fixed interval to find if it is alive. Use this to set that
+   * interval
+   *
+   * @param interval time in milliseconds
+   */
+  public void setAliveCheckInterval(int interval) {
+    if (interval <= 0) {
+      throw new IllegalArgumentException("Alive check interval must be " +
+              "positive, specified value = " + interval);
+    }
+    this.interval = interval;
+  }
+
+  private void startAliveCheckExecutor() {
+    // double-checked locking, but it's OK because we don't *do* anything with aliveCheckExecutor
+    // if it's not null.
+    if (aliveCheckExecutor == null) {
+      synchronized (this) {
+        if (aliveCheckExecutor == null) {
+          aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(
+              new SolrjNamedThreadFactory("aliveCheckExecutor"));
+          aliveCheckExecutor.scheduleAtFixedRate(
+                  getAliveCheckRunner(new WeakReference<>(this)),
+                  this.interval, this.interval, TimeUnit.MILLISECONDS);
+        }
+      }
+    }
+  }
+
+  private static Runnable getAliveCheckRunner(final WeakReference<LBHttp2SolrClient> lbRef) {
+    return () -> {
+      LBHttp2SolrClient lb = lbRef.get();
+      if (lb != null && lb.zombieServers != null) {
+        for (ServerWrapper zombieServer : lb.zombieServers.values()) {
+          lb.checkAZombieServer(zombieServer);
+        }
+      }
+    };
+  }
+
+  /**
+   * Return the HttpClient this instance uses.
+   */
+  private Http2SolrClient getHttpClient() {
+    return httpClient;
+  }
+
+  public ResponseParser getParser() {
+    return parser;
+  }
+
+  /**
+   * Changes the {@link ResponseParser} that will be used for the internal
+   * SolrServer objects.
+   *
+   * @param parser Default Response Parser chosen to parse the response if the parser
+   *               were not specified as part of the request.
+   * @see SolrRequest#getResponseParser()
+   */
+  public void setParser(ResponseParser parser) {
+    this.parser = parser;
+  }
+
+  /**
+   * Changes the {@link RequestWriter} that will be used for the internal
+   * SolrServer objects.
+   *
+   * @param requestWriter Default RequestWriter, used to encode requests sent to the server.
+   */
+  public void setRequestWriter(RequestWriter requestWriter) {
+    this.requestWriter = requestWriter;
+  }
+  
+  public RequestWriter getRequestWriter() {
+    return requestWriter;
+  }
+  
+  @Override
+  protected void finalize() throws Throwable {
+    try {
+      if(this.aliveCheckExecutor!=null)
+        this.aliveCheckExecutor.shutdownNow();
+    } finally {
+      super.finalize();
+    }
+  }
+
+  // defaults
+  private static final int CHECK_INTERVAL = 60 * 1000; //1 minute between checks
+  private static final int NONSTANDARD_PING_LIMIT = 5;  // number of times we'll ping dead servers not in the server list
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttp2SolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttp2SolrClient.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttp2SolrClient.java
new file mode 100644
index 0000000..5cd7989
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttp2SolrClient.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.QuickPatchThreadsFilter;
+import org.apache.solr.SolrIgnoredThreadsFilter;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.embedded.JettyConfig;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrResponseBase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.util.TimeOut;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test for LBHttp2SolrClient
+ *
+ * @since solr 1.4
+ */
+@Slow
+@ThreadLeakFilters(defaultFilters = true, filters = {
+    SolrIgnoredThreadsFilter.class,
+    QuickPatchThreadsFilter.class
+})
+public class TestLBHttp2SolrClient extends SolrTestCaseJ4 {
+
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  SolrInstance[] solr = new SolrInstance[3];
+  Http2SolrClient httpClient;
+
+  // TODO: fix this test to not require FSDirectory
+  static String savedFactory;
+
+  @BeforeClass
+  public static void beforeClass() {
+    savedFactory = System.getProperty("solr.DirectoryFactory");
+    System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory");
+    System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
+  }
+
+  @AfterClass
+  public static void afterClass() {
+    if (savedFactory == null) {
+      System.clearProperty("solr.directoryFactory");
+    } else {
+      System.setProperty("solr.directoryFactory", savedFactory);
+    }
+    System.clearProperty("tests.shardhandler.randomSeed");
+  }
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    httpClient = new Http2SolrClient.Builder().connectionTimeout(500).idleTimeout(500).build();
+
+    for (int i = 0; i < solr.length; i++) {
+      solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i).toFile(), 0);
+      solr[i].setUp();
+      solr[i].startJetty();
+      addDocs(solr[i]);
+    }
+  }
+
+  private void addDocs(SolrInstance solrInstance) throws IOException, SolrServerException {
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.addField("id", i);
+      doc.addField("name", solrInstance.name);
+      docs.add(doc);
+    }
+    SolrResponseBase resp;
+    try (HttpSolrClient client = getHttpSolrClient(solrInstance.getUrl())) {
+      resp = client.add(docs);
+      assertEquals(0, resp.getStatus());
+      resp = client.commit();
+      assertEquals(0, resp.getStatus());
+    }
+  }
+
+  @Override
+  public void tearDown() throws Exception {
+    for (SolrInstance aSolr : solr) {
+      if (aSolr != null)  {
+        aSolr.tearDown();
+      }
+    }
+    httpClient.close();
+    super.tearDown();
+  }
+
+
+  public void testSimple() throws Exception {
+    String[] s = new String[solr.length];
+    for (int i = 0; i < solr.length; i++) {
+      s[i] = solr[i].getUrl();
+    }
+    LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, s);
+    client.setAliveCheckInterval(500);
+    SolrQuery solrQuery = new SolrQuery("*:*");
+    Set<String> names = new HashSet<>();
+    QueryResponse resp = null;
+    for (String ignored : s) {
+      resp = client.query(solrQuery);
+      assertEquals(10, resp.getResults().getNumFound());
+      names.add(resp.getResults().get(0).getFieldValue("name").toString());
+    }
+    assertEquals(3, names.size());
+
+    // Kill a server and test again
+    solr[1].jetty.stop();
+    solr[1].jetty = null;
+    names.clear();
+    for (String ignored : s) {
+      resp = client.query(solrQuery);
+      assertEquals(10, resp.getResults().getNumFound());
+      names.add(resp.getResults().get(0).getFieldValue("name").toString());
+    }
+    assertEquals(2, names.size());
+    assertFalse(names.contains("solr1"));
+
+    // Start the killed server once again
+    solr[1].startJetty();
+    // Wait for the alive check to complete
+    Thread.sleep(1200);
+    names.clear();
+    for (String ignored : s) {
+      resp = client.query(solrQuery);
+      assertEquals(10, resp.getResults().getNumFound());
+      names.add(resp.getResults().get(0).getFieldValue("name").toString());
+    }
+    assertEquals(3, names.size());
+  }
+
+  private LBHttp2SolrClient getLBHttp2SolrClient(Http2SolrClient httpClient, String... s) {
+    return new LBHttp2SolrClient(httpClient, s);
+  }
+
+  public void testTwoServers() throws Exception {
+    LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, solr[0].getUrl(), solr[1].getUrl());
+    client.setAliveCheckInterval(500);
+    SolrQuery solrQuery = new SolrQuery("*:*");
+    QueryResponse resp = null;
+    solr[0].jetty.stop();
+    solr[0].jetty = null;
+    resp = client.query(solrQuery);
+    String name = resp.getResults().get(0).getFieldValue("name").toString();
+    Assert.assertEquals("solr/collection11", name);
+    resp = client.query(solrQuery);
+    name = resp.getResults().get(0).getFieldValue("name").toString();
+    Assert.assertEquals("solr/collection11", name);
+    solr[1].jetty.stop();
+    solr[1].jetty = null;
+    solr[0].startJetty();
+    Thread.sleep(1200);
+    try {
+      resp = client.query(solrQuery);
+    } catch(SolrServerException e) {
+      // try again after a pause in case the error is lack of time to start server
+      Thread.sleep(3000);
+      resp = client.query(solrQuery);
+    }
+    name = resp.getResults().get(0).getFieldValue("name").toString();
+    Assert.assertEquals("solr/collection10", name);
+  }
+
+  public void testReliability() throws Exception {
+    String[] s = new String[solr.length];
+    for (int i = 0; i < solr.length; i++) {
+      s[i] = solr[i].getUrl();
+    }
+
+    LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, s);
+    client.setAliveCheckInterval(500);
+
+    // Kill a server and test again
+    solr[1].jetty.stop();
+    solr[1].jetty = null;
+
+    // query the servers
+    for (String value : s)
+      client.query(new SolrQuery("*:*"));
+
+    // Start the killed server once again
+    solr[1].startJetty();
+    // Wait for the alive check to complete
+    waitForServer(30, client, 3, solr[1].name);
+  }
+  
+  // wait maximum ms for serverName to come back up
+  private void waitForServer(int maxSeconds, LBHttp2SolrClient client, int nServers, String serverName) throws Exception {
+    final TimeOut timeout = new TimeOut(maxSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+    while (! timeout.hasTimedOut()) {
+      QueryResponse resp;
+      try {
+        resp = client.query(new SolrQuery("*:*"));
+      } catch (Exception e) {
+        log.warn("", e);
+        continue;
+      }
+      String name = resp.getResults().get(0).getFieldValue("name").toString();
+      if (name.equals(serverName))
+        return;
+      
+      Thread.sleep(500);
+    }
+  }
+  
+  private static class SolrInstance {
+    String name;
+    File homeDir;
+    File dataDir;
+    File confDir;
+    int port;
+    JettySolrRunner jetty;
+
+    public SolrInstance(String name, File homeDir, int port) {
+      this.name = name;
+      this.homeDir = homeDir;
+      this.port = port;
+
+      dataDir = new File(homeDir + "/collection1", "data");
+      confDir = new File(homeDir + "/collection1", "conf");
+    }
+
+    public String getHomeDir() {
+      return homeDir.toString();
+    }
+
+    public String getUrl() {
+      return buildUrl(port, "/solr/collection1");
+    }
+
+    public String getSchemaFile() {
+      return "solrj/solr/collection1/conf/schema-replication1.xml";
+    }
+
+    public String getConfDir() {
+      return confDir.toString();
+    }
+
+    public String getDataDir() {
+      return dataDir.toString();
+    }
+
+    public String getSolrConfigFile() {
+      return "solrj/solr/collection1/conf/solrconfig-slave1.xml";
+    }
+
+    public String getSolrXmlFile() {
+      return "solrj/solr/solr.xml";
+    }
+
+
+    public void setUp() throws Exception {
+      homeDir.mkdirs();
+      dataDir.mkdirs();
+      confDir.mkdirs();
+
+      FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
+
+      File f = new File(confDir, "solrconfig.xml");
+      FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
+      f = new File(confDir, "schema.xml");
+      FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f);
+      Files.createFile(homeDir.toPath().resolve("collection1/core.properties"));
+    }
+
+    public void tearDown() throws Exception {
+      if (jetty != null) jetty.stop();
+      IOUtils.rm(homeDir.toPath());
+    }
+
+    public void startJetty() throws Exception {
+
+      Properties props = new Properties();
+      props.setProperty("solrconfig", "bad_solrconfig.xml");
+      props.setProperty("solr.data.dir", getDataDir());
+
+      JettyConfig jettyConfig = JettyConfig.builder(buildJettyConfig("/solr")).setPort(port).build();
+
+      jetty = new JettySolrRunner(getHomeDir(), props, jettyConfig);
+      jetty.start();
+      int newPort = jetty.getLocalPort();
+      if (port != 0 && newPort != port) {
+        fail("TESTING FAILURE: could not grab requested port.");
+      }
+      this.port = newPort;
+//      System.out.println("waiting.........");
+//      Thread.sleep(5000);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/51b2a1be/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index 4bb9a29..dfea26a 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -338,7 +338,11 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
       JettySolrRunner j = createJetty(jettyHomeFile, null, null, getSolrConfigFile(), getSchemaFile());
       jettys.add(j);
       clients.add(createNewSolrClient(j.getLocalPort()));
-      String shardStr = buildUrl(j.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME;
+      String shardStr = buildUrl(j.getLocalPort());
+
+      if (shardStr.endsWith("/")) shardStr += DEFAULT_TEST_CORENAME;
+      else shardStr += "/" + DEFAULT_TEST_CORENAME;
+
       shardsArr[i] = shardStr;
       sb.append(shardStr);
     }


Mime
View raw message