lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject [24/26] lucene-solr:starburst: The Star Burst Upgrade - a work in progress - the branch gets replaced often.
Date Sun, 29 Jul 2018 15:08:46 GMT
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
index 28c3cdf..ab3d51e 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
@@ -16,13 +16,16 @@
  */
 package org.apache.solr.client.solrj.embedded;
 
-import org.eclipse.jetty.servlet.ServletHolder;
-
-import javax.servlet.Filter;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.TreeMap;
 
+import javax.servlet.Filter;
+
+import org.apache.solr.client.solrj.util.SolrInternalHttpClient;
+import org.apache.solr.client.solrj.util.SolrQueuedThreadPool;
+import org.eclipse.jetty.servlet.ServletHolder;
+
 public class JettyConfig {
 
   public final int port;
@@ -38,9 +41,13 @@ public class JettyConfig {
   public final Map<Class<? extends Filter>, String> extraFilters;
 
   public final SSLConfig sslConfig;
+  
+  public SolrInternalHttpClient httpClient;
+  
+  public SolrQueuedThreadPool qtp;
 
   private JettyConfig(int port, String context, boolean stopAtShutdown, Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
-                      Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig) {
+                      Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig, SolrInternalHttpClient httpClient, SolrQueuedThreadPool qtp) {
     this.port = port;
     this.context = context;
     this.stopAtShutdown = stopAtShutdown;
@@ -48,6 +55,8 @@ public class JettyConfig {
     this.extraServlets = extraServlets;
     this.extraFilters = extraFilters;
     this.sslConfig = sslConfig;
+    this.httpClient = httpClient;
+    this.qtp = qtp;
   }
 
   public static Builder builder() {
@@ -62,6 +71,8 @@ public class JettyConfig {
     builder.extraServlets = other.extraServlets;
     builder.extraFilters = other.extraFilters;
     builder.sslConfig = other.sslConfig;
+    builder.httpClient = other.httpClient;
+    builder.qtp = other.qtp;
     return builder;
   }
 
@@ -74,6 +85,8 @@ public class JettyConfig {
     Map<ServletHolder, String> extraServlets = new TreeMap<>();
     Map<Class<? extends Filter>, String> extraFilters = new LinkedHashMap<>();
     SSLConfig sslConfig = null;
+    SolrInternalHttpClient httpClient;
+    SolrQueuedThreadPool qtp;
 
     public Builder setPort(int port) {
       this.port = port;
@@ -81,6 +94,7 @@ public class JettyConfig {
     }
 
     public Builder setContext(String context) {
+      //assert !context.endsWith("/") : context;
       this.context = context;
       return this;
     }
@@ -121,9 +135,19 @@ public class JettyConfig {
       this.sslConfig = sslConfig;
       return this;
     }
+    
+    public Builder withHttpClient(SolrInternalHttpClient httpClient) {
+      this.httpClient = httpClient;
+      return this;
+    }
+    
+    public Builder withJettyQtp(SolrQueuedThreadPool qtp) {
+      this.qtp = qtp;
+      return this;
+    }
 
     public JettyConfig build() {
-      return new JettyConfig(port, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig);
+      return new JettyConfig(port, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig, httpClient, qtp);
     }
 
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index 23a8dc1..4d1a7e7 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -16,16 +16,6 @@
  */
 package org.apache.solr.client.solrj.embedded;
 
-import javax.servlet.DispatcherType;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
-import javax.servlet.FilterConfig;
-import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.net.MalformedURLException;
@@ -37,14 +27,30 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
+import javax.servlet.DispatcherType;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
 import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.SolrHttpClientScheduler;
+import org.apache.solr.client.solrj.util.SolrInternalHttpClient;
+import org.apache.solr.client.solrj.util.SolrQueuedThreadPool;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.servlet.SolrQoSFilter;
+import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -60,7 +66,7 @@ import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.servlet.Source;
 import org.eclipse.jetty.util.component.LifeCycle;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.eclipse.jetty.util.thread.Scheduler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
@@ -82,6 +88,9 @@ public class JettySolrRunner {
 
   FilterHolder dispatchFilter;
   FilterHolder debugFilter;
+  FilterHolder qosFilter;
+  
+  private static Scheduler scheduler = new SolrHttpClientScheduler("JettySolrRunnerScheduler", true, null, new ThreadGroup("JettySolrRunnerScheduler"), 1);
 
   private boolean waitOnSolr = false;
   private int jettyPort = -1;
@@ -97,6 +106,12 @@ public class JettySolrRunner {
   private static final String excludePatterns = "/css/.+,/js/.+,/img/.+,/tpl/.+";
   
   private int proxyPort = -1;
+  
+  private SolrInternalHttpClient httpClient;
+
+  private SolrQueuedThreadPool qtp;
+  
+  private static AtomicInteger count = new AtomicInteger();
 
   public static class DebugFilter implements Filter {
     public final static Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -204,21 +219,38 @@ public class JettySolrRunner {
     this.solrHome = solrHome;
     this.config = config;
     this.nodeProperties = nodeProperties;
+    
+    this.httpClient = config.httpClient;
+    assert this.httpClient != null;
+    if (this.httpClient == null) {
+      httpClient = new SolrInternalHttpClient(JettySolrRunner.class.getSimpleName() + "-internal");
+      try {
+        httpClient.start();
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+    assert config.qtp != null;
+    this.qtp = config.qtp;
+
+    if (this.qtp == null) {
+      qtp = new SolrQueuedThreadPool(10000, 0, 1000, null, new ThreadGroup("JettyServer-" + JettySolrRunner.class.getSimpleName() + "-internal"));
+      qtp.setName("JettyServer-" + JettySolrRunner.class.getSimpleName() + "-internal");
+      //qtp.setReservedThreads(0);
+      //qtp.setStopTimeout((int) TimeUnit.MINUTES.toMillis(1));
+    }
 
     this.init(this.config.port);
   }
   
   private void init(int port) {
+    int cnt = count.incrementAndGet();
 
-    QueuedThreadPool qtp = new QueuedThreadPool();
-    qtp.setMaxThreads(THREAD_POOL_MAX_THREADS);
-    qtp.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
-    qtp.setStopTimeout((int) TimeUnit.MINUTES.toMillis(1));
     server = new Server(qtp);
-    server.manage(qtp);
+
     server.setStopAtShutdown(config.stopAtShutdown);
 
-    if (System.getProperty("jetty.testMode") != null) {
+    if (false) {
       // if this property is true, then jetty will be configured to use SSL
       // leveraging the same system properties as java to specify
       // the keystore/truststore if they are set unless specific config
@@ -250,11 +282,56 @@ public class JettySolrRunner {
       server.setConnectors(new Connector[] {connector});
       server.setSessionIdManager(new DefaultSessionIdManager(server, new Random()));
     } else {
-      ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory());
-      connector.setPort(port);
+//      ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory());
+//      connector.setPort(port);
+//      connector.setSoLingerTime(-1);
+//      connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
+//      connector.setHost("127.0.0.1");
+//      server.setConnectors(new Connector[] {connector});
+      
+      SslContextFactory sslcontext = SSLConfig.createContextFactory(config.sslConfig);
+      
+      ServerConnector connector;
+      HttpConfiguration configuration = new HttpConfiguration();
+      HTTP2ServerConnectionFactory h2;
+      
+      if (sslcontext != null) {
+        System.out.println("USE SSL");
+        sslcontext.setProvider("Conscrypt");
+        configuration.setSecureScheme("https");
+        configuration.addCustomizer(new SecureRequestCustomizer());
+        h2 = new HTTP2ServerConnectionFactory(configuration);
+        h2.setInputBufferSize(16384);
+        connector = new ServerConnector(server, null, scheduler, null, 1, 1, new SslConnectionFactory(sslcontext, "h2"), h2);
+        //    new HttpConnectionFactory(configuration));
+      } else {
+        h2 = new HTTP2ServerConnectionFactory(configuration);
+        h2.setInputBufferSize(16384);
+        connector = new ServerConnector(server, null, scheduler, null, 1, 1,  h2);
+      }
+      
+      // HTTP/2 Connection Factory
+      h2.setMaxConcurrentStreams(1500);
+
+      //connector.addConnectionFactory(h2);
+      connector.setDefaultProtocol("h2");
+      connector.setReuseAddress(true);
       connector.setSoLingerTime(-1);
+      connector.setPort(port);
+      connector.setHost("127.0.0.1");
       connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
-      server.setConnectors(new Connector[] {connector});
+      connector.setStopTimeout(0); // for test speed we shutdown hard and fast // nocommit
+      // TODO: SSL - 
+     // NegotiatingServerConnectionFactory.checkProtocolNegotiationAvailable();
+//      ALPNServerConnectionFactory alpn = new ALPNServerConnectionFactory();
+//      alpn.setDefaultProtocol("h2");
+//      ServerConnector http2Connector = new ServerConnector(server, 1, 1, h2); // use 1 selector and acceptor instead of cpu based default - can be a bunch of jetties
+//      http2Connector.setReuseAddress(true);
+//      http2Connector.setSoLingerTime(-1);
+//      http2Connector.setPort(port);
+//      http2Connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
+//      http2Connector.setHost("127.0.0.1");
+        server.addConnector(connector);
     }
 
     // Initialize the servlets
@@ -291,21 +368,27 @@ public class JettySolrRunner {
 
         logger.info("Jetty properties: {}", nodeProperties);
 
-        debugFilter = root.addFilter(DebugFilter.class, "*", EnumSet.of(DispatcherType.REQUEST) );
+        debugFilter = root.addFilter(DebugFilter.class, "*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC));
         extraFilters = new LinkedList<>();
         for (Class<? extends Filter> filterClass : config.extraFilters.keySet()) {
           extraFilters.add(root.addFilter(filterClass, config.extraFilters.get(filterClass),
-              EnumSet.of(DispatcherType.REQUEST)));
+              EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC)));
         }
 
         for (ServletHolder servletHolder : config.extraServlets.keySet()) {
           String pathSpec = config.extraServlets.get(servletHolder);
           root.addServlet(servletHolder, pathSpec);
         }
+        
+        qosFilter = root.getServletHandler().newFilterHolder(Source.EMBEDDED);
+        qosFilter.setHeldClass(SolrQoSFilter.class);
+        root.addFilter(qosFilter, "*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC));
+
         dispatchFilter = root.getServletHandler().newFilterHolder(Source.EMBEDDED);
         dispatchFilter.setHeldClass(SolrDispatchFilter.class);
         dispatchFilter.setInitParameter("excludePatterns", excludePatterns);
-        root.addFilter(dispatchFilter, "*", EnumSet.of(DispatcherType.REQUEST));
+        root.addFilter(dispatchFilter, "*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ASYNC));
+        
       }
 
       @Override
@@ -425,8 +508,17 @@ public class JettySolrRunner {
     Map<String, String> prevContext = MDC.getCopyOfContextMap();
     MDC.clear();
     try {
+
       Filter filter = dispatchFilter.getFilter();
 
+      if (config.httpClient == null) {
+        IOUtils.closeQuietly(httpClient);
+      }
+      
+      if (config.qtp == null) {
+        IOUtils.closeQuietly(qtp);
+      }
+      
       server.stop();
 
       if (server.getState().equals(Server.FAILED)) {
@@ -438,7 +530,7 @@ public class JettySolrRunner {
         }
       }
 
-      server.join();
+      //server.join();
     } finally {
       if (prevContext != null)  {
         MDC.setContextMap(prevContext);
@@ -508,13 +600,14 @@ public class JettySolrRunner {
   }
 
   public SolrClient newClient() {
-    return new HttpSolrClient.Builder(getBaseUrl().toString()).build();
+    return new Http2SolrClient.Builder(getBaseUrl().toString()).withHttpClient(httpClient).build();
   }
   
   public SolrClient newClient(int connectionTimeoutMillis, int socketTimeoutMillis) {
-    return new HttpSolrClient.Builder(getBaseUrl().toString())
-        .withConnectionTimeout(connectionTimeoutMillis)
-        .withSocketTimeout(socketTimeoutMillis)
+    return new Http2SolrClient.Builder(getBaseUrl().toString())
+        //.withConnectionTimeout(connectionTimeoutMillis)
+        //.withSocketTimeout(socketTimeoutMillis)
+        .withHttpClient(httpClient)
         .build();
   }
 
@@ -533,6 +626,7 @@ public class JettySolrRunner {
     public void service(HttpServletRequest req, HttpServletResponse res)
         throws IOException {
       res.sendError(404, "Can not find: " + req.getRequestURI());
+      //res.setStatus(404);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
index 62c9024..32a8ba6 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
@@ -52,7 +52,8 @@ public class SSLConfig {
   
   /** All other settings on this object are ignored unless this is true */
   public boolean isSSLMode() {
-    return useSsl;
+    return false; // nocommit fix ssl support: javax.net.ssl.SSLException: Unable to parse TLS packet header
+    // return useSsl;
   }
   
   public boolean isClientAuthMode() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/ActiveReplicaWatcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ActiveReplicaWatcher.java b/solr/core/src/java/org/apache/solr/cloud/ActiveReplicaWatcher.java
index c6bd807..978505f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ActiveReplicaWatcher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ActiveReplicaWatcher.java
@@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory;
 
 /**
  * Watch for replicas to become {@link org.apache.solr.common.cloud.Replica.State#ACTIVE}. Watcher is
- * terminated (its {@link #onStateChanged(Set, DocCollection)} method returns false) when all listed
+ * terminated (its {@link #onStateChanged(boolean, Set, DocCollection)} method returns false) when all listed
  * replicas become active.
  * <p>Additionally, the provided {@link SolrCloseableLatch} instance can be used to await
  * for all listed replicas to become active.</p>
@@ -114,7 +114,8 @@ public class ActiveReplicaWatcher implements CollectionStateWatcher {
 
   // synchronized due to SOLR-11535
   @Override
-  public synchronized boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
+  public synchronized boolean onStateChanged(boolean closing, Set<String> liveNodes, DocCollection collectionState) {
+    if (closing) return false;
     log.debug("-- onStateChanged@" + Long.toHexString(hashCode()) + ": replicaIds=" + replicaIds + ", solrCoreNames=" + solrCoreNames +
         (latch != null ? "\nlatch count=" + latch.getCount() : "") +
         "\ncollectionState=" + collectionState);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index 5ee5bdd..8bf0a60 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.cloud;
 
+import static org.apache.solr.common.params.CommonParams.ID;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
@@ -26,6 +28,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -62,8 +65,6 @@ import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.params.CommonParams.ID;
-
 public abstract class ElectionContext implements Closeable {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   final String electionPath;
@@ -100,7 +101,7 @@ public abstract class ElectionContext implements Closeable {
     }
   }
 
-  abstract void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStartMs) throws KeeperException, InterruptedException, IOException;
+  abstract void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException, IOException;
 
   public void checkIfIamLeaderFired() {}
 
@@ -170,7 +171,7 @@ class ShardLeaderElectionContextBase extends ElectionContext {
   }
   
   @Override
-  void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStartMs)
+  void runLeaderProcess(boolean weAreReplacement)
       throws KeeperException, InterruptedException, IOException {
     // register as leader - if an ephemeral is already there, wait to see if it goes away
     
@@ -300,7 +301,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
    * weAreReplacement: has someone else been the leader already?
    */
   @Override
-  void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStart) throws KeeperException,
+  void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
  InterruptedException, IOException {
     String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
     ActionThrottle lt;
@@ -386,12 +387,13 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
         
         if (weAreReplacement) {
           // wait a moment for any floating updates to finish
-          try {
-            Thread.sleep(2500);
-          } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e);
-          }
+          // nocommit
+//          try {
+//            Thread.sleep(2500);
+//          } catch (InterruptedException e) {
+//            Thread.currentThread().interrupt();
+//            throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e);
+//          }
         }
 
         PeerSync.PeerSyncResult result = null;
@@ -478,7 +480,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
                 "without being up-to-date with the previous leader", coreNodeName);
             zkController.getShardTerms(collection, shardId).setTermEqualsToLeader(coreNodeName);
           }
-          super.runLeaderProcess(weAreReplacement, 0);
+          super.runLeaderProcess(weAreReplacement);
           try (SolrCore core = cc.getCore(coreName)) {
             if (core != null) {
               core.getCoreDescriptor().getCloudDescriptor().setLeader(true);
@@ -533,27 +535,30 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
    * false if otherwise
    */
   private boolean waitForEligibleBecomeLeaderAfterTimeout(ZkShardTerms zkShardTerms, String coreNodeName, int timeout) throws InterruptedException {
-    long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS);
-    while (!isClosed && !cc.isShutDown()) {
-      if (System.nanoTime() > timeoutAt) {
-        return true;
-      }
-      if (replicasWithHigherTermParticipated(zkShardTerms, coreNodeName)) {
-        log.info("Can't become leader, other replicas with higher term participated in leader election");
+
+    try {
+      zkStateReader.waitForState(collection, timeout, TimeUnit.MILLISECONDS, (n, c) -> {
+        boolean r = replicasWithHigherTermParticipated(c, n, zkShardTerms, coreNodeName);
+        if (r) {
+          return true;
+        }
         return false;
-      }
-      Thread.sleep(500L);
+      });
+    } catch (TimeoutException e) {
+      return true;
     }
+    
     return false;
   }
 
   /**
    * Do other replicas with higher term participated in the election
+   * @param c docCollection
+   * @param liveNodes liveNodes
    * @return true if other replicas with higher term participated in the election, false if otherwise
    */
-  private boolean replicasWithHigherTermParticipated(ZkShardTerms zkShardTerms, String coreNodeName) {
-    ClusterState clusterState = zkController.getClusterState();
-    DocCollection docCollection = clusterState.getCollectionOrNull(collection);
+  private boolean replicasWithHigherTermParticipated(DocCollection c, Set<String> liveNodes, ZkShardTerms zkShardTerms, String coreNodeName) {
+    DocCollection docCollection = c;
     Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId);
     if (slices == null) return false;
 
@@ -563,7 +568,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
     for (Replica replica : slices.getReplicas()) {
       if (replica.getName().equals(coreNodeName)) continue;
 
-      if (clusterState.getLiveNodes().contains(replica.getNodeName())) {
+      if (liveNodes.contains(replica.getNodeName())) {
         long otherTerm = zkShardTerms.getTerm(replica.getName());
         boolean isOtherReplicaRecovering = zkShardTerms.isRecovering(replica.getName());
 
@@ -830,7 +835,7 @@ final class OverseerElectionContext extends ElectionContext {
   }
 
   @Override
-  void runLeaderProcess(boolean weAreReplacement, int pauseBeforeStartMs) throws KeeperException,
+  void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
       InterruptedException {
     log.info("I am going to be the leader {}", id);
     final String id = leaderSeqPath
@@ -839,14 +844,7 @@ final class OverseerElectionContext extends ElectionContext {
 
     zkClient.makePath(leaderPath, Utils.toJSON(myProps),
         CreateMode.EPHEMERAL, true);
-    if(pauseBeforeStartMs >0){
-      try {
-        Thread.sleep(pauseBeforeStartMs);
-      } catch (InterruptedException e) {
-        Thread.interrupted();
-        log.warn("Wait interrupted ", e);
-      }
-    }
+
     if (!overseer.getZkController().isClosed() && !overseer.getZkController().getCoreContainer().isShutDown()) {
       overseer.start(id);
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
index 46f3c88..aa4b97e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
@@ -167,7 +167,7 @@ public  class LeaderElector {
   // TODO: get this core param out of here
   protected void runIamLeaderProcess(final ElectionContext context, boolean weAreReplacement) throws KeeperException,
       InterruptedException, IOException {
-    context.runLeaderProcess(weAreReplacement,0);
+    context.runLeaderProcess(weAreReplacement);
   }
   
   /**
@@ -281,11 +281,6 @@ public  class LeaderElector {
             throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
                 "", e);
           }
-          try {
-            Thread.sleep(50);
-          } catch (InterruptedException e2) {
-            Thread.currentThread().interrupt();
-          }
         }
 
       } catch (KeeperException.NoNodeException e) {
@@ -297,11 +292,6 @@ public  class LeaderElector {
               "", e);
         }
         cont = true;
-        try {
-          Thread.sleep(50);
-        } catch (InterruptedException e2) {
-          Thread.currentThread().interrupt();
-        }
       }
     }
     checkIfIamLeader(context, replacement);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java
index 071bfbf..aed5157 100644
--- a/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderInitiatedRecoveryThread.java
@@ -16,9 +16,15 @@
  */
 package org.apache.solr.cloud;
 
+import java.lang.invoke.MethodHandles;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.List;
+
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.conn.ConnectTimeoutException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
@@ -30,17 +36,11 @@ import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
-import org.apache.zookeeper.KeeperException;
 import org.apache.solr.util.RTimer;
+import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.lang.invoke.MethodHandles;
-import java.net.ConnectException;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.List;
-
 /**
  * Background daemon thread that tries to send the REQUESTRECOVERY to a downed
  * replica; used by a shard leader to nag a replica into recovering after the
@@ -213,9 +213,10 @@ public class LeaderInitiatedRecoveryThread extends Thread {
         log.info("Asking core={} coreNodeName={} on " + recoveryUrl + " to recover", coreNeedingRecovery, replicaCoreNodeName);
       }
 
-      try (HttpSolrClient client = new HttpSolrClient.Builder(recoveryUrl)
-          .withConnectionTimeout(15000)
-          .withSocketTimeout(60000)
+      // nocommit no shared exec, but this going away anyway...
+      try (Http2SolrClient client = new Http2SolrClient.Builder(recoveryUrl)
+          //.withConnectionTimeout(15000)
+          //.withSocketTimeout(60000)
           .build()) {
         try {
           client.request(recoverRequestCmd);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
index 966497b..358b1d1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
@@ -25,17 +25,19 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient.HttpUriRequestResponse;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient.Abortable;
 import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.SolrResponseBase;
+import org.apache.solr.client.solrj.util.SolrInternalHttpClient;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.cloud.DocCollection;
@@ -118,9 +120,13 @@ public class RecoveryStrategy implements Runnable, Closeable {
   private int retries;
   private boolean recoveringAfterStartup;
   private CoreContainer cc;
-  private volatile HttpUriRequest prevSendPreRecoveryHttpUriRequest;
+ //private volatile HttpUriRequest prevSendPreRecoveryHttpUriRequest;
   private final Replica.Type replicaType;
 
+  private SolrInternalHttpClient httpClient;
+
+  private volatile Abortable abortable;
+
   protected RecoveryStrategy(CoreContainer cc, CoreDescriptor cd, RecoveryListener recoveryListener) {
     this.cc = cc;
     this.coreName = cd.getName();
@@ -130,6 +136,7 @@ public class RecoveryStrategy implements Runnable, Closeable {
     baseUrl = zkController.getBaseUrl();
     coreZkNodeName = cd.getCloudDescriptor().getCoreNodeName();
     replicaType = cd.getCloudDescriptor().getReplicaType();
+    httpClient = cc.getUpdateShardHandler().getDefaultHttpClient();
   }
 
   final public int getWaitForUpdatesWithStaleStatePauseMilliSeconds() {
@@ -168,9 +175,12 @@ public class RecoveryStrategy implements Runnable, Closeable {
   @Override
   final public void close() {
     close = true;
-    if (prevSendPreRecoveryHttpUriRequest != null) {
-      prevSendPreRecoveryHttpUriRequest.abort();
+    try {
+      abortable.abort();
+    } catch (NullPointerException e) {
+      // expected
     }
+    
     LOG.warn("Stopping recovery for core=[{}] coreNodeName=[{}]", coreName, coreZkNodeName);
   }
 
@@ -263,9 +273,7 @@ public class RecoveryStrategy implements Runnable, Closeable {
 
   final private void commitOnLeader(String leaderUrl) throws SolrServerException,
       IOException {
-    try (HttpSolrClient client = new HttpSolrClient.Builder(leaderUrl)
-        .withConnectionTimeout(30000)
-        .build()) {
+    try (Http2SolrClient client = new Http2SolrClient.Builder(leaderUrl).solrInternal().build()) {
       UpdateRequest ureq = new UpdateRequest();
       ureq.setParams(new ModifiableSolrParams());
       ureq.getParams().set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
@@ -552,7 +560,7 @@ public class RecoveryStrategy implements Runnable, Closeable {
             .getSlice(cloudDesc.getShardId());
             
         try {
-          prevSendPreRecoveryHttpUriRequest.abort();
+          //prevSendPreRecoveryHttpUriRequest.abort();
         } catch (NullPointerException e) {
           // okay
         }
@@ -739,7 +747,7 @@ public class RecoveryStrategy implements Runnable, Closeable {
       Replica leaderReplica = null;
 
       if (isClosed()) {
-        return leaderReplica;
+        return null;
       }
 
       try {
@@ -753,11 +761,13 @@ public class RecoveryStrategy implements Runnable, Closeable {
       if (leaderReplica.getCoreUrl().equals(ourUrl)) {
         return leaderReplica;
       }
+      
+      if (isClosed()) {
+        return null;
+      }
 
-      try (HttpSolrClient httpSolrClient = new HttpSolrClient.Builder(leaderReplica.getCoreUrl())
-          .withSocketTimeout(1000)
-          .withConnectionTimeout(1000)
-          .build()) {
+      try (Http2SolrClient httpSolrClient = new Http2SolrClient.Builder(leaderReplica.getCoreUrl()).solrInternal()
+          .withHttpClient(httpClient).build()) {
         SolrPingResponse resp = httpSolrClient.ping();
         return leaderReplica;
       } catch (IOException e) {
@@ -849,15 +859,27 @@ public class RecoveryStrategy implements Runnable, Closeable {
     int conflictWaitMs = zkController.getLeaderConflictResolveWait();
     // timeout after 5 seconds more than the max timeout (conflictWait + 3 seconds) on the server side
     int readTimeout = conflictWaitMs + 8000;
-    try (HttpSolrClient client = new HttpSolrClient.Builder(leaderBaseUrl).build()) {
-      client.setConnectionTimeout(10000);
-      client.setSoTimeout(readTimeout);
-      HttpUriRequestResponse mrr = client.httpUriRequest(prepCmd);
-      prevSendPreRecoveryHttpUriRequest = mrr.httpUriRequest;
+    try (Http2SolrClient client = new Http2SolrClient.Builder(leaderBaseUrl).withHttpClient(httpClient)
+        .solrInternal().idleConnectionTimeout(readTimeout).build()) {
 
+      AtomicBoolean failed = new AtomicBoolean(false);
       LOG.info("Sending prep recovery command to [{}]; [{}]", leaderBaseUrl, prepCmd.toString());
-
-      mrr.future.get();
+      abortable = client.abortableRequest(prepCmd, new Http2SolrClient.OnComplete<SolrResponseBase>() {
+        
+        @Override
+        public void onSuccess(SolrResponseBase result) {}
+        
+        @Override
+        public void onFailure(Throwable e) {
+          failed.set(true);
+        }
+      });
+      abortable.get();
+     // mrr.future.get();
+      if (failed.get()) {
+        // nocommit
+        throw new RuntimeException("sendPrepRecoveryCmd failed");
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
index 4368af9..b2cf730 100644
--- a/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
@@ -16,15 +16,17 @@
  */
 package org.apache.solr.cloud;
 
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
 
-import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.SolrResponse;
 import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient.OnComplete;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -44,8 +46,6 @@ import org.apache.solr.update.UpdateShardHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.params.CommonParams.DISTRIB;
-
 public class SyncStrategy {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
@@ -54,12 +54,10 @@ public class SyncStrategy {
   private final ShardHandler shardHandler;
 
   private volatile boolean isClosed;
-  
-  private final HttpClient client;
 
-  private final ExecutorService updateExecutor;
-  
   private final List<RecoveryRequest> recoveryRequests = new ArrayList<>();
+
+  private Http2SolrClient solrClient;
   
   private static class RecoveryRequest {
     ZkNodeProps leaderProps;
@@ -69,9 +67,8 @@ public class SyncStrategy {
   
   public SyncStrategy(CoreContainer cc) {
     UpdateShardHandler updateShardHandler = cc.getUpdateShardHandler();
-    client = updateShardHandler.getDefaultHttpClient();
     shardHandler = cc.getShardHandlerFactory().getShardHandler();
-    updateExecutor = updateShardHandler.getUpdateExecutor();
+    solrClient = updateShardHandler.getUpdateSolrClient();
   }
   
   private static class ShardCoreRequest extends ShardRequest {
@@ -281,32 +278,29 @@ public class SyncStrategy {
     }
   }
   
-  private void requestRecovery(final ZkNodeProps leaderProps, final String baseUrl, final String coreName) throws SolrServerException, IOException {
-    Thread thread = new Thread() {
-      {
-        setDaemon(true);
-      }
-      @Override
-      public void run() {
-        RequestRecovery recoverRequestCmd = new RequestRecovery();
-        recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
-        recoverRequestCmd.setCoreName(coreName);
-        
-        try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl)
-            .withHttpClient(SyncStrategy.this.client)
-            .withConnectionTimeout(30000)
-            .withSocketTimeout(120000)
-            .build()) {
-          client.request(recoverRequestCmd);
-        } catch (Throwable t) {
-          SolrException.log(log, ZkCoreNodeProps.getCoreUrl(leaderProps) + ": Could not tell a replica to recover", t);
-          if (t instanceof Error) {
-            throw (Error) t;
-          }
-        }
+  private void requestRecovery(final ZkNodeProps leaderProps, final String baseUrl, final String coreName)
+      throws SolrServerException, IOException {
+
+    RequestRecovery recoverRequestCmd = new RequestRecovery();
+    recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
+    recoverRequestCmd.setCoreName(coreName);
+    try {
+      solrClient.request(recoverRequestCmd, null, new OnComplete<SolrResponse>() {
+
+        @Override
+        public void onSuccess(SolrResponse result) {}
+
+        @Override
+        public void onFailure(Throwable e) {
+          // nocommit
+        }});
+    } catch (Throwable t) {
+      SolrException.log(log, ZkCoreNodeProps.getCoreUrl(leaderProps) + ": Could not tell a replica to recover", t);
+      if (t instanceof Error) {
+        throw (Error) t;
       }
-    };
-    updateExecutor.execute(thread);
+    }
+
   }
   
   public static ModifiableSolrParams params(String... params) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
index 2df87a0..580e486 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
@@ -16,7 +16,9 @@
  */
 package org.apache.solr.cloud;
 
-import javax.xml.parsers.ParserConfigurationException;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -28,7 +30,8 @@ import java.util.List;
 import java.util.concurrent.TimeoutException;
 import java.util.regex.Pattern;
 
-import com.google.common.annotations.VisibleForTesting;
+import javax.xml.parsers.ParserConfigurationException;
+
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.HelpFormatter;
@@ -47,8 +50,7 @@ import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.xml.sax.SAXException;
 
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
+import com.google.common.annotations.VisibleForTesting;
 
 public class ZkCLI {
   
@@ -193,6 +195,7 @@ public class ZkCLI {
         zkServer.start();
       }
       SolrZkClient zkClient = null;
+      CoreContainer cc = null;
       try {
         zkClient = new SolrZkClient(zkServerAddress, 30000, 30000,
             () -> {
@@ -205,7 +208,7 @@ public class ZkCLI {
             System.exit(1);
           }
 
-          CoreContainer cc = new CoreContainer(solrHome);
+          cc = new CoreContainer(solrHome);
 
           if(!ZkController.checkChrootPath(zkServerAddress, true)) {
             stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
@@ -364,6 +367,9 @@ public class ZkCLI {
         if (zkClient != null) {
           zkClient.close();
         }
+        if (cc != null) {
+          cc.shutdown();
+        }
       }
     } catch (ParseException exp) {
       stdout.println("Unexpected exception:" + exp.getMessage());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 367a131..f221a35 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -16,6 +16,15 @@
  */
 package org.apache.solr.cloud;
 
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+
 import java.io.File;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -51,15 +60,13 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 
-import com.google.common.base.Strings;
 import org.apache.commons.lang.StringUtils;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
 import org.apache.solr.cloud.overseer.OverseerAction;
 import org.apache.solr.cloud.overseer.SliceMutator;
 import org.apache.solr.common.SolrException;
@@ -120,14 +127,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
 
-import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import com.google.common.base.Strings;
 
 /**
  * Handle ZooKeeper interactions.
@@ -910,8 +910,8 @@ public class ZkController {
 
     CountDownLatch latch = new CountDownLatch(collectionsWithLocalReplica.size());
     for (String collectionWithLocalReplica : collectionsWithLocalReplica) {
-      zkStateReader.registerCollectionStateWatcher(collectionWithLocalReplica, (liveNodes, collectionState) -> {
-        if (collectionState == null)  return false;
+      zkStateReader.registerCollectionStateWatcher(collectionWithLocalReplica, (closing, liveNodes, collectionState) -> {
+        if (closing || collectionState == null)  return false;
         boolean foundStates = true;
         for (CoreDescriptor coreDescriptor : cc.getCoreDescriptors()) {
           if (coreDescriptor.getCloudDescriptor().getCollectionName().equals(collectionWithLocalReplica))  {
@@ -1578,38 +1578,38 @@ public class ZkController {
     }
   }
 
-  private void waitForCoreNodeName(CoreDescriptor descriptor) {
-    int retryCount = 320;
+  private void waitForCoreNodeName(CoreDescriptor cd) {
     log.debug("look for our core node name");
-    while (retryCount-- > 0) {
-      final DocCollection docCollection = zkStateReader.getClusterState()
-          .getCollectionOrNull(descriptor.getCloudDescriptor().getCollectionName());
-      if (docCollection != null && docCollection.getSlicesMap() != null) {
-        final Map<String, Slice> slicesMap = docCollection.getSlicesMap();
+    AtomicReference<String> errorMessage = new AtomicReference<>();
+    try {
+      zkStateReader.waitForState(cd.getCollectionName(), 320, TimeUnit.SECONDS, (n, c) -> {
+        if (c == null)
+          return false;
+        final Map<String,Slice> slicesMap = c.getSlicesMap();
         for (Slice slice : slicesMap.values()) {
           for (Replica replica : slice.getReplicas()) {
-            // TODO: for really large clusters, we could 'index' on this
 
             String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
             String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
 
             String msgNodeName = getNodeName();
-            String msgCore = descriptor.getName();
+            String msgCore = cd.getName();
 
             if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
-              descriptor.getCloudDescriptor()
+              cd.getCloudDescriptor()
                   .setCoreNodeName(replica.getName());
-              getCoreContainer().getCoresLocator().persist(getCoreContainer(), descriptor);
-              return;
+              getCoreContainer().getCoresLocator().persist(getCoreContainer(), cd);
+              return true;
             }
           }
         }
-      }
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
+        return false;
+      });
+    } catch (TimeoutException | InterruptedException e) {
+      String error = errorMessage.get();
+      if (error == null)
+        error = "Timeout waiting for collection state.";
+      throw new NotInClusterStateException(ErrorCode.SERVER_ERROR, error);
     }
   }
 
@@ -1819,9 +1819,10 @@ public class ZkController {
         log.info("Replica " + myCoreNodeName +
             " NOT in leader-initiated recovery, need to wait for leader to see down state.");
 
-        try (HttpSolrClient client = new Builder(leaderBaseUrl)
-            .withConnectionTimeout(15000)
-            .withSocketTimeout(120000)
+        try (Http2SolrClient client = new Http2SolrClient.Builder(leaderBaseUrl)
+            //.withConnectionTimeout(15000)
+            //.withSocketTimeout(120000)
+            .withHttpClient(cc.getUpdateShardHandler().getDefaultHttpClient())
             .build()) {
           WaitForState prepCmd = new WaitForState();
           prepCmd.setCoreName(leaderCoreName);
@@ -1872,7 +1873,7 @@ public class ZkController {
             }
           }
         } catch (IOException e) {
-          SolrException.log(log, "Error closing HttpSolrClient", e);
+          SolrException.log(log, "Error closing Http2SolrClient", e);
         }
       }
     }
@@ -2736,7 +2737,11 @@ public class ZkController {
 
     @Override
     // synchronized due to SOLR-11535
-    public synchronized boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
+    public synchronized boolean onStateChanged(boolean closing, Set<String> liveNodes, DocCollection collectionState) {
+      if (closing) {
+        return true;
+      }
+      
       if (getCoreContainer().getCoreDescriptor(coreName) == null) return true;
 
       boolean replicaRemoved = getReplicaOrNull(collectionState, shard, coreNodeName) == null;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
index 4dbc059..cae305c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java
@@ -16,6 +16,12 @@
  */
 package org.apache.solr.cloud.api.collections;
 
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -44,12 +50,6 @@ import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-
 
 public class DeleteReplicaCmd implements Cmd {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/api/collections/LeaderRecoveryWatcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/LeaderRecoveryWatcher.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/LeaderRecoveryWatcher.java
index a80fdc0..b879229 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/LeaderRecoveryWatcher.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/LeaderRecoveryWatcher.java
@@ -53,8 +53,8 @@ public class LeaderRecoveryWatcher implements CollectionStateWatcher {
   }
 
   @Override
-  public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
-    if (collectionState == null) { // collection has been deleted - don't wait
+  public boolean onStateChanged(boolean closing, Set<String> liveNodes, DocCollection collectionState) {
+    if (closing || collectionState == null) { // collection has been deleted - don't wait
       latch.countDown();
       return true;
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index fed9254..e1c2ca3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -16,6 +16,56 @@
  */
 package org.apache.solr.cloud.api.collections;
 
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
+import static org.apache.solr.common.cloud.DocCollection.SNITCH;
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ALIASPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.BACKUP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATEALIAS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESNAPSHOT;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEALIAS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETENODE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESNAPSHOT;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MAINTAINROUTEDALIAS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATESTATEFORMAT;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOCK_COLL_TASK;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOCK_REPLICA_TASK;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOCK_SHARD_TASK;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.REBALANCELEADERS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.REPLACENODE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.RESTORE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.UTILIZENODE;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.util.Utils.makeMap;
+
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -30,8 +80,9 @@ import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 
-import com.google.common.collect.ImmutableMap;
 import org.apache.commons.lang.StringUtils;
 import org.apache.solr.client.solrj.SolrResponse;
 import org.apache.solr.client.solrj.SolrServerException;
@@ -40,8 +91,8 @@ import org.apache.solr.client.solrj.cloud.DistributedQueue;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
 import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
 import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient.RemoteSolrException;
 import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.UpdateResponse;
@@ -53,6 +104,7 @@ import org.apache.solr.cloud.OverseerSolrResponse;
 import org.apache.solr.cloud.OverseerTaskProcessor;
 import org.apache.solr.cloud.Stats;
 import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkController.NotInClusterStateException;
 import org.apache.solr.cloud.overseer.OverseerAction;
 import org.apache.solr.common.SolrCloseable;
 import org.apache.solr.common.SolrException;
@@ -79,6 +131,7 @@ import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.SuppressForbidden;
 import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.common.util.Utils;
+import org.apache.solr.handler.component.HttpShardHandlerFactory;
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.handler.component.ShardHandlerFactory;
 import org.apache.solr.handler.component.ShardRequest;
@@ -92,23 +145,7 @@ import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY;
-import static org.apache.solr.common.cloud.DocCollection.SNITCH;
-import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.util.Utils.makeMap;
+import com.google.common.collect.ImmutableMap;
 
 /**
  * A {@link OverseerMessageHandler} that handles Collections API related
@@ -496,11 +533,13 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
   }
 
 
-  static UpdateResponse softCommit(String url) throws SolrServerException, IOException {
+  UpdateResponse softCommit(String url) throws SolrServerException, IOException {
 
-    try (HttpSolrClient client = new HttpSolrClient.Builder(url)
-        .withConnectionTimeout(30000)
-        .withSocketTimeout(120000)
+    // nocommit: use update instead of shard handler?
+    try (Http2SolrClient client = new Http2SolrClient.Builder(url)
+        //.withConnectionTimeout(30000)
+        //.withSocketTimeout(120000)
+        .withHttpClient(((HttpShardHandlerFactory)shardHandlerFactory).getHttpClient())
         .build()) {
       UpdateRequest ureq = new UpdateRequest();
       ureq.setParams(new ModifiableSolrParams());
@@ -510,31 +549,36 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
   }
 
   String waitForCoreNodeName(String collectionName, String msgNodeName, String msgCore) {
-    int retryCount = 320;
-    while (retryCount-- > 0) {
-      final DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
-      if (docCollection != null && docCollection.getSlicesMap() != null) {
-        Map<String,Slice> slicesMap = docCollection.getSlicesMap();
+    AtomicReference<String> replicaName = new AtomicReference<>();
+    AtomicReference<String> errorMessage = new AtomicReference<>();
+    try {
+      zkStateReader.waitForState(collectionName, 320, TimeUnit.SECONDS, (n, c) -> {
+        if (c == null)
+          return false;
+        final Map<String,Slice> slicesMap = c.getSlicesMap();
         for (Slice slice : slicesMap.values()) {
+          System.out.println("slice: " + slice.getName());
           for (Replica replica : slice.getReplicas()) {
-            // TODO: for really large clusters, we could 'index' on this
-
+            System.out.println("replica:" + replica);
             String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
             String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
-
+            System.out.println("nodeName:" + nodeName + " >" + msgNodeName);
+            System.out.println("core:" + core + " >" + msgCore);
             if (nodeName.equals(msgNodeName) && core.equals(msgCore)) {
-              return replica.getName();
+              replicaName.set(replica.getName());
+              return true;
             }
           }
         }
-      }
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-      }
+        return false;
+      });
+    } catch (TimeoutException | InterruptedException e) {
+      String error = errorMessage.get();
+      if (error == null)
+        error = "Timeout waiting to find coreNodeName.";
+      throw new NotInClusterStateException(ErrorCode.SERVER_ERROR, error);
     }
-    throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName");
+    return replicaName.get();
   }
 
   void waitForNewShard(String collectionName, String sliceName) throws KeeperException, InterruptedException {
@@ -671,33 +715,41 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
   }
 
   Map<String, Replica> waitToSeeReplicasInState(String collectionName, Collection<String> coreNames) throws InterruptedException {
-    Map<String, Replica> result = new HashMap<>();
-    TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, timeSource);
-    while (true) {
-      DocCollection coll = zkStateReader.getClusterState().getCollection(collectionName);
-      for (String coreName : coreNames) {
-        if (result.containsKey(coreName)) continue;
-        for (Slice slice : coll.getSlices()) {
-          for (Replica replica : slice.getReplicas()) {
-            if (coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) {
-              result.put(coreName, replica);
-              break;
+    AtomicReference<Map<String, Replica>> result = new AtomicReference<>();
+    AtomicReference<String> errorMessage = new AtomicReference<>();
+    try {
+      zkStateReader.waitForState(collectionName, 30, TimeUnit.SECONDS, (n, c) -> {
+        if (c == null)
+          return false;
+        Map<String, Replica> r = new HashMap<>();
+        for (String coreName : coreNames) {
+          if (r.containsKey(coreName)) continue;
+          for (Slice slice : c.getSlices()) {
+            for (Replica replica : slice.getReplicas()) {
+              if (coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) {
+                r.put(coreName, replica);
+                break;
+              }
             }
           }
         }
-      }
-      
-      if (result.size() == coreNames.size()) {
-        return result;
-      } else {
-        log.debug("Expecting {} cores but found {}", coreNames, result);
-      }
-      if (timeout.hasTimedOut()) {
-        throw new SolrException(ErrorCode.SERVER_ERROR, "Timed out waiting to see all replicas: " + coreNames + " in cluster state. Last state: " + coll);
-      }
-      
-      Thread.sleep(100);
+        
+        if (r.size() == coreNames.size()) {
+          result.set(r);
+          return true;
+        } else {
+          errorMessage.set("Timed out waiting to see all replicas: " + coreNames + " in cluster state. Last state: " + c);
+          return false;
+        }
+
+      });
+    } catch (TimeoutException | InterruptedException e) {
+      String error = errorMessage.get();
+      if (error == null)
+        error = "Timeout waiting for collection state.";
+      throw new NotInClusterStateException(ErrorCode.SERVER_ERROR, error);
     }
+    return result.get();
   }
 
   ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete)

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
index d08b519..9eccac5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
@@ -18,6 +18,10 @@
 package org.apache.solr.cloud.api.collections;
 
 
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -48,10 +52,6 @@ import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-
 public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index ca7c2d6..7ec964c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -18,6 +18,20 @@
 package org.apache.solr.cloud.api.collections;
 
 
+import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonParams.NAME;
+
 import java.lang.invoke.MethodHandles;
 import java.net.URI;
 import java.util.ArrayList;
@@ -59,20 +73,6 @@ import org.apache.solr.handler.component.ShardHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
-import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-import static org.apache.solr.common.params.CommonParams.NAME;
-
 public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/BlobRepository.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index 0cf6dfb..f96ec5b 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -34,9 +34,9 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 
 import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient.SimpleResponse;
+import org.apache.solr.client.solrj.util.SolrInternalHttpClient;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.DocCollection;
@@ -45,7 +45,6 @@ import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.SimplePostTool;
 import org.apache.zookeeper.server.ByteBufferInputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -157,22 +156,17 @@ public class BlobRepository {
     Replica replica = getSystemCollReplica();
     String url = replica.getStr(BASE_URL_PROP) + "/" + CollectionAdminParams.SYSTEM_COLL + "/blob/" + key + "?wt=filestream";
 
-    HttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
-    HttpGet httpGet = new HttpGet(url);
+    SolrInternalHttpClient httpClient = coreContainer.getUpdateShardHandler().getDefaultHttpClient();
     ByteBuffer b;
-    HttpResponse response = null;
     HttpEntity entity = null;
-    try {
-      response = httpClient.execute(httpGet);
-      entity = response.getEntity();
-      int statusCode = response.getStatusLine().getStatusCode();
-      if (statusCode != 200) {
+    try (Http2SolrClient solrClient = new Http2SolrClient.Builder(url).withHttpClient(httpClient).build()) {
+      SimpleResponse rsp = solrClient.httpGet(url);
+
+      if (rsp.status != 200) {
         throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such blob or version available: " + key);
       }
+      b = ByteBuffer.wrap(rsp.bytes);
 
-      try (InputStream is = entity.getContent()) {
-        b = SimplePostTool.inputStreamToByteArray(is);
-      }
     } catch (Exception e) {
       if (e instanceof SolrException) {
         throw (SolrException) e;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/CloudConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CloudConfig.java b/solr/core/src/java/org/apache/solr/core/CloudConfig.java
index 6248b45..fecb21b 100644
--- a/solr/core/src/java/org/apache/solr/core/CloudConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/CloudConfig.java
@@ -127,7 +127,7 @@ public class CloudConfig {
     private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 15000;
     private static final int DEFAULT_LEADER_VOTE_WAIT = 180000;  // 3 minutes
     private static final int DEFAULT_LEADER_CONFLICT_RESOLVE_WAIT = 180000;
-    private static final int DEFAULT_CREATE_COLLECTION_ACTIVE_WAIT = 30;  // 30 seconds
+    private static final int DEFAULT_CREATE_COLLECTION_ACTIVE_WAIT = 60;  // 60 seconds
     private static final boolean DEFAULT_CREATE_COLLECTION_CHECK_LEADER_ACTIVE = false; 
  
     private static final int DEFAULT_AUTO_REPLICA_FAILOVER_WAIT_AFTER_EXPIRATION = 120000;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 37a660f..38c1583 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -16,6 +16,21 @@
  */
 package org.apache.solr.core;
 
+import static java.util.Objects.requireNonNull;
+import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
+import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
+import static org.apache.solr.common.params.CommonParams.AUTOSCALING_HISTORY_PATH;
+import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
+import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
+import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
+import static org.apache.solr.common.params.CommonParams.ZK_PATH;
+import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
+import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
+
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.Path;
@@ -36,9 +51,9 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
 import org.apache.http.auth.AuthSchemeProvider;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.config.Lookup;
@@ -53,6 +68,7 @@ import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
 import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder;
 import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder.AuthSchemeRegistryProvider;
 import org.apache.solr.client.solrj.impl.SolrHttpClientContextBuilder.CredentialsProviderProvider;
+import org.apache.solr.client.solrj.util.SolrInternalHttpClient;
 import org.apache.solr.client.solrj.util.SolrIdentifierValidator;
 import org.apache.solr.cloud.CloudDescriptor;
 import org.apache.solr.cloud.Overseer;
@@ -66,6 +82,8 @@ import org.apache.solr.common.cloud.Replica.State;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.util.ExecutorUtil;
 import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.DirectoryFactory.DirContext;
 import org.apache.solr.core.backup.repository.BackupRepository;
@@ -103,25 +121,14 @@ import org.apache.solr.update.SolrCoreState;
 import org.apache.solr.update.UpdateShardHandler;
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.apache.solr.util.OrderedExecutor;
+import org.apache.solr.util.SolrFileCleaningTracker;
 import org.apache.solr.util.stats.MetricUtils;
 import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static java.util.Objects.requireNonNull;
-import static org.apache.solr.common.params.CommonParams.AUTHC_PATH;
-import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH;
-import static org.apache.solr.common.params.CommonParams.AUTOSCALING_HISTORY_PATH;
-import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
-import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
-import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
-import static org.apache.solr.common.params.CommonParams.ZK_PATH;
-import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
-import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
 
 /**
  *
@@ -143,6 +150,12 @@ public class CoreContainer {
       this.exception = loadFailure;
     }
   }
+  
+  private final SolrFileCleaningTracker fileCleaningTracker = new SolrFileCleaningTracker();
+
+  public SolrFileCleaningTracker getFileCleaningTracker() {
+    return fileCleaningTracker;
+  }
 
   protected final Map<String, CoreLoadFailure> coreInitFailures = new ConcurrentHashMap<>();
 
@@ -159,7 +172,14 @@ public class CoreContainer {
 
   private ConfigSetService coreConfigService;
 
-  protected ZkContainer zkSys = new ZkContainer();
+  private ExecutorService executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE,
+      5L, TimeUnit.SECONDS,
+      new SynchronousQueue<>(),
+      new SolrjNamedThreadFactory(CoreContainer.class.getSimpleName() + "Executor"),
+      // the Runnable added to this executor handles all exceptions so we disable stack trace collection as an optimization
+      // see SOLR-11880 for more details
+      false);
+  protected ZkContainer zkSys = new ZkContainer(executor);
   protected ShardHandlerFactory shardHandlerFactory;
 
   private UpdateShardHandler updateShardHandler;
@@ -217,6 +237,8 @@ public class CoreContainer {
   protected AutoScalingHandler autoScalingHandler;
 
   private enum CoreInitFailedAction { fromleader, none }
+  
+  private volatile SolrInternalHttpClient httpClient;
 
   /**
    * This method instantiates a new instance of {@linkplain BackupRepository}.
@@ -316,6 +338,8 @@ public class CoreContainer {
         ExecutorUtil.newMDCAwareCachedThreadPool(
             cfg.getReplayUpdatesThreads(),
             new DefaultSolrThreadFactory("replayUpdatesExecutor")));
+    
+    assert ObjectReleaseTracker.track(this);
   }
 
   private synchronized void initializeAuthorizationPlugin(Map<String, Object> authorizationConf) {
@@ -538,12 +562,13 @@ public class CoreContainer {
         SolrMetricManager.mkName("coreContainerWorkExecutor", SolrInfoBean.Category.CONTAINER.toString(), "threadPool"));
 
     shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
+    shardHandlerFactory.setHttpClient(getHttpClient());
     if (shardHandlerFactory instanceof SolrMetricProducer) {
       SolrMetricProducer metricProducer = (SolrMetricProducer) shardHandlerFactory;
       metricProducer.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "httpShardHandler");
     }
 
-    updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
+    updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig(), getHttpClient(), executor);
     updateShardHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "updateShardHandler");
 
     solrCores.load(loader);
@@ -739,6 +764,16 @@ public class CoreContainer {
     status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
   }
 
+  private synchronized SolrInternalHttpClient getHttpClient() {
+    if (isShutDown) {
+      throw new IllegalStateException("CoreContainer is shutdown."); // nocommit this could interfere with in flight requests
+    }
+    if (httpClient == null) {
+      httpClient = new SolrInternalHttpClient(CoreContainer.class.getSimpleName(), true);;
+    }
+    return httpClient;
+  }
+
   // MetricsHistoryHandler supports both cloud and standalone configs
   private void createMetricsHistoryHandler() {
     PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
@@ -843,6 +878,7 @@ public class CoreContainer {
     }
 
     if (isZooKeeperAware()) {
+      zkSys.zkController.zkStateReader.stop();
       cancelCoreRecoveries();
       zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName());
       try {
@@ -904,10 +940,17 @@ public class CoreContainer {
           if (updateShardHandler != null) {
             updateShardHandler.close();
           }
+          IOUtils.closeQuietly(httpClient);
+          ExecutorUtil.shutdownAndAwaitTermination(executor);
         } finally {
           // we want to close zk stuff last
           zkSys.close();
         }
+        try {
+          fileCleaningTracker.exitWhenFinished();
+        } catch (Exception e) {
+          log.warn("Exception closing FileCleaningTracker", e);
+        }
       }
     }
 
@@ -931,6 +974,8 @@ public class CoreContainer {
     }
 
     org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort
+    
+    assert ObjectReleaseTracker.release(this);
   }
 
   public void cancelCoreRecoveries() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/Diagnostics.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/Diagnostics.java b/solr/core/src/java/org/apache/solr/core/Diagnostics.java
index d7d6178..1ee6847 100644
--- a/solr/core/src/java/org/apache/solr/core/Diagnostics.java
+++ b/solr/core/src/java/org/apache/solr/core/Diagnostics.java
@@ -15,13 +15,13 @@
  * limitations under the License.
  */
 package org.apache.solr.core;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.lang.invoke.MethodHandles;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class Diagnostics {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
index fab3300..a4a9e44 100644
--- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -23,9 +23,9 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.nio.file.NoSuchFileException;
-import java.util.Arrays;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/74a9b54c/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
index d17d3d6..4e0b0db 100644
--- a/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
@@ -16,6 +16,8 @@
  */
 package org.apache.solr.core;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
@@ -29,10 +31,6 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
@@ -69,7 +67,10 @@ import org.apache.solr.util.plugin.SolrCoreAware;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
 
 public class HdfsDirectoryFactory extends CachingDirectoryFactory implements SolrCoreAware, SolrMetricProducer {
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());


Mime
View raw message