lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject [31/31] lucene-solr:jira/http2: Resolved merge conflict caused by SOLR-11881
Date Fri, 10 Aug 2018 09:14:11 GMT
Resolved merge conflict caused by SOLR-11881


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f554f6d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f554f6d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f554f6d

Branch: refs/heads/jira/http2
Commit: 9f554f6de0420e39128ce366e6e8e028c2ff66e7
Parents: 51b2a1b 9c02bf0
Author: Cao Manh Dat <datcm@apache.org>
Authored: Fri Aug 10 16:13:18 2018 +0700
Committer: Cao Manh Dat <datcm@apache.org>
Committed: Fri Aug 10 16:13:18 2018 +0700

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  14 +
 .../org/apache/lucene/search/BlockMaxDISI.java  |  94 ++++
 .../lucene/search/Boolean2ScorerSupplier.java   |   4 +-
 .../org/apache/lucene/search/BooleanWeight.java |   2 +-
 .../lucene/search/DisjunctionMaxQuery.java      |   2 +-
 .../lucene/search/DisjunctionMaxScorer.java     |  80 ++--
 .../apache/lucene/search/DisjunctionScorer.java |  22 +-
 .../lucene/search/DisjunctionSumScorer.java     |   4 +-
 .../apache/lucene/search/ReqOptSumScorer.java   | 175 ++++---
 .../apache/lucene/index/TestIndexWriter.java    |   2 +
 .../lucene/search/TestDisjunctionMaxQuery.java  |  77 ++-
 .../lucene/search/TestReqOptSumScorer.java      | 241 +++++++++-
 lucene/ivy-versions.properties                  |   1 -
 .../org/apache/lucene/document/LatLonShape.java |  13 +-
 .../document/LatLonShapeBoundingBoxQuery.java   | 420 +++++------------
 .../document/LatLonShapePolygonQuery.java       | 301 ++----------
 .../lucene/document/LatLonShapeQuery.java       | 394 ++++++++++++++++
 .../document/BaseLatLonShapeTestCase.java       |  46 +-
 .../document/TestLatLonLineShapeQueries.java    |  40 +-
 .../document/TestLatLonPointShapeQueries.java   |  20 +-
 .../document/TestLatLonPolygonShapeQueries.java |  36 +-
 .../apache/lucene/document/TestLatLonShape.java |   3 +-
 .../spatial3d/geom/GeoComplexPolygon.java       |   8 +-
 .../org/apache/lucene/spatial3d/geom/Plane.java |   2 +-
 .../lucene/spatial3d/geom/GeoPolygonTest.java   |  63 ++-
 .../lucene/search/RandomApproximationQuery.java |  10 +-
 solr/CHANGES.txt                                |  25 +
 .../configsets/collection1/conf/solrconfig.xml  |  20 +-
 solr/core/ivy.xml                               |   1 -
 .../org/apache/solr/core/ConfigSetService.java  |   7 +-
 .../org/apache/solr/core/CoreContainer.java     |   3 +
 .../apache/solr/handler/ReplicationHandler.java |   4 +-
 .../handler/admin/ZookeeperStatusHandler.java   | 222 +++++++++
 .../handler/component/TermVectorComponent.java  |  89 ++--
 .../solr/handler/export/DoubleValueSortDoc.java |  93 ++++
 .../solr/handler/export/ExportWriter.java       |  13 +-
 .../solr/handler/export/QuadValueSortDoc.java   | 126 +++++
 .../solr/handler/export/SingleValueSortDoc.java |  82 ++++
 .../org/apache/solr/handler/export/SortDoc.java |   7 +-
 .../solr/handler/export/TripleValueSortDoc.java | 110 +++++
 .../apache/solr/handler/loader/XMLLoader.java   |  18 +-
 .../org/apache/solr/handler/sql/SolrSchema.java |   4 +
 .../solr/response/RetrieveFieldsOptimizer.java  |   8 +-
 .../transform/BaseEditorialTransformer.java     |   2 +
 .../apache/solr/update/SolrCmdDistributor.java  | 194 +++++---
 .../processor/DistributedUpdateProcessor.java   |  36 +-
 .../ParseDateFieldUpdateProcessorFactory.java   |  82 +++-
 .../TimeRoutedAliasUpdateProcessor.java         |   4 +-
 .../org/apache/solr/util/TestInjection.java     |  22 +-
 .../solr/collection1/conf/schema-sql.xml        |  27 ++
 .../test-files/solr/collection1/conf/schema.xml |   2 +-
 ...dd-schema-fields-update-processor-chains.xml |  20 +-
 ...lrconfig-parsing-update-processor-chains.xml |  49 +-
 .../collection1/conf/solrconfig-schemaless.xml  |  20 +-
 .../configsets/_default/conf/solrconfig.xml     |  20 +-
 .../org/apache/solr/BasicFunctionalityTest.java | 158 +++----
 .../test/org/apache/solr/CursorPagingTest.java  |  10 +-
 .../solr/TestCursorMarkWithoutUniqueKey.java    |   9 +-
 .../org/apache/solr/TestDistributedSearch.java  |  61 ++-
 .../org/apache/solr/TestTolerantSearch.java     |  18 +-
 .../solr/cloud/BasicDistributedZk2Test.java     |  30 +-
 .../solr/cloud/BasicDistributedZkTest.java      |  14 +-
 .../test/org/apache/solr/cloud/BasicZkTest.java |  14 +-
 .../solr/cloud/CollectionsAPISolrJTest.java     |  46 +-
 .../apache/solr/cloud/DeleteReplicaTest.java    |  19 +-
 .../org/apache/solr/cloud/ForceLeaderTest.java  |  10 +-
 .../apache/solr/cloud/OverseerStatusTest.java   |  17 +-
 .../org/apache/solr/cloud/SolrXmlInZkTest.java  |  10 +-
 .../solr/cloud/TestAuthenticationFramework.java |  10 +-
 .../solr/cloud/TestCloudDeleteByQuery.java      |  11 +-
 .../apache/solr/cloud/TestCloudInspectUtil.java |  19 +-
 .../apache/solr/cloud/TestConfigSetsAPI.java    |  11 +-
 .../solr/cloud/TestConfigSetsAPIZkFailure.java  |  14 +-
 .../solr/cloud/TestDownShardTolerantSearch.java |  17 +-
 .../TestLeaderInitiatedRecoveryThread.java      |  64 ++-
 .../cloud/TestPullReplicaErrorHandling.java     |  13 +-
 .../TestSolrCloudWithDelegationTokens.java      |  25 +-
 .../TestSolrCloudWithSecureImpersonation.java   | 101 ++--
 .../cloud/TestTolerantUpdateProcessorCloud.java | 238 +++++-----
 .../org/apache/solr/cloud/TestZkChroot.java     |  23 +-
 .../test/org/apache/solr/cloud/ZkCLITest.java   |  18 +-
 .../org/apache/solr/cloud/ZkSolrClientTest.java |  59 +--
 .../autoscaling/AutoScalingHandlerTest.java     |  43 +-
 .../org/apache/solr/core/TestLazyCores.java     |  15 +
 .../org/apache/solr/handler/TestSQLHandler.java | 243 ++++++++--
 .../solr/handler/admin/MetricsHandlerTest.java  |   4 +-
 .../admin/ZookeeperStatusHandlerTest.java       |  85 ++++
 .../component/QueryElevationComponentTest.java  |  30 ++
 .../component/TermVectorComponentTest.java      | 340 +++++++------
 .../apache/solr/update/AddBlockUpdateTest.java  | 148 ++++++
 .../solr/update/MockingHttp2SolrClient.java     |  87 +++-
 .../solr/update/SolrCmdDistributorTest.java     | 472 +++++++++++++++----
 ...dSchemaFieldsUpdateProcessorFactoryTest.java |  23 +-
 .../ParsingFieldUpdateProcessorsTest.java       | 157 +++---
 solr/example/files/conf/solrconfig.xml          |  20 +-
 solr/licenses/joda-time-2.2.jar.sha1            |   1 -
 solr/licenses/joda-time-LICENSE-ASL.txt         | 202 --------
 solr/licenses/joda-time-NOTICE.txt              |   5 -
 .../configsets/_default/conf/solrconfig.xml     |  20 +-
 solr/solr-ref-guide/src/cloud-screens.adoc      |   5 +
 .../src/images/cloud-screens/cloud-zkstatus.png | Bin 0 -> 175359 bytes
 solr/solr-ref-guide/src/schemaless-mode.adoc    |  20 +-
 ...solrcloud-autoscaling-auto-add-replicas.adoc |   6 +-
 .../src/solrcloud-autoscaling-overview.adoc     |   2 +-
 ...olrcloud-autoscaling-policy-preferences.adoc | 167 +++++--
 .../src/transforming-result-documents.adoc      |   9 +-
 .../cloud/autoscaling/AddReplicaSuggester.java  |  10 +-
 .../client/solrj/cloud/autoscaling/Clause.java  |  18 +-
 .../cloud/autoscaling/MoveReplicaSuggester.java |   3 +-
 .../solrj/cloud/autoscaling/NodeVariable.java   |   5 +-
 .../client/solrj/cloud/autoscaling/Policy.java  |   2 +-
 .../solrj/cloud/autoscaling/PolicyHelper.java   |  38 +-
 .../solrj/cloud/autoscaling/RangeVal.java       |  10 +-
 .../solrj/cloud/autoscaling/ReplicaCount.java   |   6 +-
 .../solrj/cloud/autoscaling/ReplicaInfo.java    |  17 +-
 .../solrj/cloud/autoscaling/Suggester.java      |  14 +-
 .../solrj/cloud/autoscaling/VariableBase.java   |   1 +
 .../solrj/cloud/autoscaling/Violation.java      |  10 +-
 .../solr/client/solrj/impl/Http2SolrClient.java |  14 +-
 .../solrj/impl/SolrClientNodeStateProvider.java |   2 +-
 .../org/apache/solr/client/solrj/io/Lang.java   |   1 +
 .../solr/client/solrj/io/eval/KnnEvaluator.java |  18 +-
 .../solrj/io/eval/KnnRegressionEvaluator.java   | 194 ++++++++
 .../solrj/io/eval/MinMaxScaleEvaluator.java     |   2 +-
 .../client/solrj/io/eval/PolyFitEvaluator.java  |   7 +-
 .../client/solrj/io/eval/PredictEvaluator.java  |  30 +-
 .../solr/client/solrj/util/ClientUtils.java     |  20 +-
 .../solr/common/ConditionalMapWriter.java       |  75 +++
 .../java/org/apache/solr/common/MapWriter.java  |   6 +
 .../apache/solr/common/params/CommonParams.java |   1 +
 .../java/org/apache/solr/common/util/XML.java   |  98 +---
 .../solrj/cloud/autoscaling/TestPolicy.java     |  16 +-
 .../solrj/cloud/autoscaling/TestPolicy2.java    | 240 ++++++++++
 .../apache/solr/client/solrj/io/TestLang.java   |   2 +-
 .../solrj/io/stream/MathExpressionTest.java     |  86 +++-
 .../apache/solr/cloud/SolrCloudTestCase.java    |  19 +
 .../org/apache/solr/util/BaseTestHarness.java   |  23 +-
 solr/webapp/web/css/angular/cloud.css           |  71 ++-
 solr/webapp/web/css/angular/menu.css            |   1 +
 solr/webapp/web/index.html                      |   1 +
 solr/webapp/web/js/angular/controllers/cloud.js |  42 +-
 solr/webapp/web/js/angular/services.js          |   6 +
 solr/webapp/web/partials/cloud.html             |  59 +++
 143 files changed, 5253 insertions(+), 2442 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/lucene/ivy-versions.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 7437042,d5aafec..710f03f
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@@ -17,15 -17,28 +17,18 @@@
  package org.apache.solr.update;
  
  
 -import java.io.Closeable;
  import java.io.IOException;
 -import java.io.InputStream;
  import java.lang.invoke.MethodHandles;
  import java.net.ConnectException;
+ import java.net.SocketException;
+ import java.net.SocketTimeoutException;
  import java.util.ArrayList;
 -import java.util.Collections;
 -import java.util.HashSet;
  import java.util.List;
 -import java.util.Set;
 -import java.util.concurrent.CompletionService;
 -import java.util.concurrent.ExecutorCompletionService;
 -import java.util.concurrent.Future;
 -import org.apache.http.HttpResponse;
 +import java.util.concurrent.Phaser;
 +
+ import org.apache.http.NoHttpResponseException;
 -import org.apache.solr.client.solrj.SolrClient;
  import org.apache.solr.client.solrj.SolrServerException;
 -import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 -import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; // jdoc
 -import org.apache.solr.client.solrj.impl.HttpSolrClient;
 +import org.apache.solr.client.solrj.impl.Http2SolrClient;
  import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
  import org.apache.solr.client.solrj.request.UpdateRequest;
  import org.apache.solr.common.SolrException;
@@@ -43,28 -56,133 +46,27 @@@ import org.slf4j.LoggerFactory
  /**
   * Used for distributing commands from a shard leader to its replicas.
   */
 -public class SolrCmdDistributor implements Closeable {
 +public class SolrCmdDistributor {
-   private static final int MAX_RETRIES_ON_FORWARD = 25;
    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 -  
 -  private StreamingSolrClients clients;
 -  private boolean finished = false; // see finish()
  
 -  private int retryPause = 500;
 -  
    private final List<Error> allErrors = new ArrayList<>();
 -  private final List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
 -  
 -  private final CompletionService<Object> completionService;
 -  private final Set<Future<Object>> pending = new HashSet<>();
 -  
 -  public static interface AbortCheck {
 -    public boolean abortCheck();
 -  }
 -  
 +  private Http2SolrClient client;
 +  private Phaser pendingTasksPhaser = new Phaser(1);
-   private int maxRetriesOnForward = MAX_RETRIES_ON_FORWARD;
++  private int retryPause = 500;
 +
    public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
 -    this.clients = new StreamingSolrClients(updateShardHandler);
 -    this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
 +    this.client = updateShardHandler.getUpdateOnlyHttpClient();
    }
--  
-   public SolrCmdDistributor(Http2SolrClient client, int maxRetriesOnForward) {
 -  public SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
 -    this.clients = clients;
++
++  public SolrCmdDistributor(Http2SolrClient client, int retryPause) {
 +    this.client = client;
-     this.maxRetriesOnForward = maxRetriesOnForward;
+     this.retryPause = retryPause;
 -    completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
    }
    
 -  public void finish() {    
 -    try {
 -      assert ! finished : "lifecycle sanity check";
 -      finished = true;
 -      
 -      blockAndDoRetries();
 -    } finally {
 -      clients.shutdown();
 -    }
 +  public void finish() {
 +    blockUntilFinished();
    }
 -  
 -  public void close() {
 -    clients.shutdown();
 -  }
 -
 -  private void doRetriesIfNeeded() {
 -    // NOTE: retries will be forwards to a single url
 -    
 -    List<Error> errors = new ArrayList<>(this.errors);
 -    errors.addAll(clients.getErrors());
 -    List<Error> resubmitList = new ArrayList<>();
 -    
 -    if (log.isInfoEnabled() && errors.size() > 0) {
 -      log.info("SolrCmdDistributor found {} errors", errors.size());
 -    }
 -    
 -    if (log.isDebugEnabled() && errors.size() > 0) {
 -      StringBuilder builder = new StringBuilder("SolrCmdDistributor found:");
 -      int maxErrorsToShow = 10;
 -      for (Error e:errors) {
 -        if (maxErrorsToShow-- <= 0) break;
 -        builder.append("\n" + e);
 -      }
 -      if (errors.size() > 10) {
 -        builder.append("\n... and ");
 -        builder.append(errors.size() - 10);
 -        builder.append(" more");
 -      }
 -      log.debug(builder.toString());
 -    }
  
 -    for (Error err : errors) {
 -      try {
 -        String oldNodeUrl = err.req.node.getUrl();
 -        
 -        /*
 -         * if this is a retryable request we may want to retry, depending on the error we received and
 -         * the number of times we have already retried
 -         */
 -        boolean isRetry = err.req.shouldRetry(err);
 -        
 -        if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
 -            err.e);
 -        
 -        // this can happen in certain situations such as close
 -        if (isRetry) {
 -          err.req.retries++;
 -
 -          if (err.req.node instanceof ForwardNode) {
 -            SolrException.log(SolrCmdDistributor.log, "forwarding update to "
 -                + oldNodeUrl + " failed - retrying ... retries: "
 -                + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
 -                + err.req.cmd.toString() + " params:"
 -                + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
 -          } else {
 -            SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
 -                + oldNodeUrl + " failed - retrying ... retries: "
 -                + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
 -                + err.req.cmd.toString() + " params:"
 -                + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
 -          }
 -          try {
 -            Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
 -          } catch (InterruptedException e) {
 -            Thread.currentThread().interrupt();
 -            log.warn(null, e);
 -          }
 -          resubmitList.add(err);
 -        } else {
 -          allErrors.add(err);
 -        }
 -      } catch (Exception e) {
 -        // continue on
 -        log.error("Unexpected Error while doing request retries", e);
 -      }
 -    }
 -    
 -    clients.clearErrors();
 -    this.errors.clear();
 -    for (Error err : resubmitList) {
 -      submit(err.req, false);
 -    }
 -    
 -    if (resubmitList.size() > 0) {
 -      blockAndDoRetries();
 -    }
 -  }
 -  
    public void distribDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
      distribDelete(cmd, nodes, params, false, null, null);
    }
@@@ -73,6 -191,10 +75,10 @@@
                              RollupRequestReplicationTracker rollupTracker,
                              LeaderRequestReplicationTracker leaderTracker) throws IOException {
      
+     if (!cmd.isDeleteById()) {
 -      blockAndDoRetries(); // For DBQ, flush all writes before submitting
++      blockUntilFinished(); // For DBQ, flush all writes before submitting
+     }
+     
      for (Node node : nodes) {
        UpdateRequest uReq = new UpdateRequest();
        uReq.setParams(params);
@@@ -153,96 -302,36 +158,97 @@@
            + req.node.getUrl() + " retry:"
            + req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
      }
 -    
 -    if (isCommit) {
 -      // a commit using ConncurrentUpdateSolrServer is not async,
 -      // so we make it async to prevent commits from happening
 -      // serially across multiple nodes
 -      pending.add(completionService.submit(() -> {
 -        doRequest(req);
 -        return null;
 -      }));
 +
 +    try {
 +      req.uReq.setBasePath(req.node.getUrl());
 +      if (req.synchronous) {
 +        NamedList rsp = client.request(req.uReq);
 +        req.trackRequestResult(rsp, true);
 +        pendingTasksPhaser.arriveAndDeregister();
 +      } else {
 +        //TODO write add cmds in single outputstream
 +        client.request(req.uReq, null, new Http2SolrClient.OnComplete<NamedList>() {
 +          @Override
 +          public void onSuccess(NamedList result) {
 +            req.trackRequestResult(result, true);
 +            pendingTasksPhaser.arriveAndDeregister();
 +          }
 +
 +          @Override
 +          public void onFailure(Throwable t) {
 +            handleAndRetry(req, t, isCommit);
 +          }
 +        });
 +      }
 +    } catch (Exception e) {
 +      handleAndRetry(req, e, isCommit);
 +    }
 +  }
 +
 +  private void handleAndRetry(Req req, Throwable t, boolean isCommit) {
 +    SolrException.log(log, t);
 +    Error error = new Error();
 +    error.t = t;
 +    error.req = req;
 +    if (t instanceof SolrException) {
 +      error.statusCode = ((SolrException) t).code();
 +    }
 +    if (checkRetry(error)) {
 +      submit0(req, isCommit);
      } else {
 -      doRequest(req);
 +      req.trackRequestResult(null, false);
 +      allErrors.add(error);
 +      pendingTasksPhaser.arriveAndDeregister();
      }
    }
 -  
 -  private void doRequest(final Req req) {
 +
 +  private boolean checkRetry(Error err) {
-     String oldNodeUrl = err.req.node.getUrl();
- 
-     // if there is a retry url, we want to retry...
-     boolean isRetry = err.req.node.checkRetry();
- 
-     boolean doRetry = false;
-     int rspCode = err.statusCode;
- 
-     if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
-         err.t);
- 
-     // this can happen in certain situations such as close
-     if (isRetry) {
-       if (rspCode == 404 || rspCode == 403 || rspCode == 503) {
-         doRetry = true;
-       }
- 
-       // if it's a connect exception, lets try again
-       if (err.t instanceof SolrServerException) {
-         if (((SolrServerException) err.t).getRootCause() instanceof ConnectException) {
-           doRetry = true;
-         }
-       }
++    log.info("SolrCmdDistributor got error", err);
+     try {
 -      SolrClient solrClient = clients.getSolrClient(req);
 -      solrClient.request(req.uReq);
 -    } catch (Exception e) {
 -      SolrException.log(log, e);
 -      Error error = new Error();
 -      error.e = e;
 -      error.req = req;
 -      if (e instanceof SolrException) {
 -        error.statusCode = ((SolrException) e).code();
++      String oldNodeUrl = err.req.node.getUrl();
 +
-       if (err.t instanceof ConnectException) {
-         doRetry = true;
-       }
++      /*
++       * if this is a retryable request we may want to retry, depending on the error we received and
++       * the number of times we have already retried
++       */
++      boolean isRetry = err.req.shouldRetry(err);
++      if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.t);
 +
-       if (err.req.retries < maxRetriesOnForward && doRetry) {
++      // this can happen in certain situations such as close
++      if (isRetry) {
 +        err.req.retries++;
 +
-         SolrException.log(SolrCmdDistributor.log, "forwarding update to "
-             + oldNodeUrl + " failed - retrying ... retries: "
-             + err.req.retries + " " + err.req.cmd.toString() + " params:"
-             + err.req.uReq.getParams() + " rsp:" + rspCode, err.t);
- 
++        if (err.req.node instanceof ForwardNode) {
++          SolrException.log(SolrCmdDistributor.log, "forwarding update to "
++              + oldNodeUrl + " failed - retrying ... retries: "
++              + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
++              + err.req.cmd.toString() + " params:"
++              + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
++        } else {
++          SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
++              + oldNodeUrl + " failed - retrying ... retries: "
++              + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
++              + err.req.cmd.toString() + " params:"
++              + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
++        }
++        try {
++          Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
++        } catch (InterruptedException e) {
++          Thread.currentThread().interrupt();
++          log.warn(null, e);
++        }
 +        return true;
 +      } else {
 +        return false;
        }
-     } else {
 -      errors.add(error);
++    } catch (Exception e) {
++      // continue on
++      log.error("Unexpected Error while doing request retries", e);
++      // avoid infinite loop
 +      return false;
      }
    }
 -  
 +
    public static class Req {
      public Node node;
      public UpdateRequest uReq;
@@@ -267,6 -356,16 +273,16 @@@
        this.leaderTracker = leaderTracker;
      }
      
+     /**
+      * @return true if this request should be retried after receiving a particular error
+      *         false otherwise
+      */
 -    public boolean shouldRetry(Error err) {
++    boolean shouldRetry(Error err) {
+       boolean isRetry = node.checkRetry(err);
+       isRetry &= uReq.getDeleteQuery() == null || uReq.getDeleteQuery().isEmpty(); //Don't retry DBQs 
+       return isRetry && retries < node.getMaxRetries();
+     }
+     
      public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("SolrCmdDistributor$Req: cmd=").append(cmd.toString());
@@@ -380,9 -511,33 +405,33 @@@
      }
  
      @Override
-     public boolean checkRetry() {
+     public boolean checkRetry(Error err) {
+       if (!retry) return false;
 -      
++
+       if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+         return true;
+       }
 -      
++
+       // if it's a connect exception, lets try again
 -      if (err.e instanceof SolrServerException) {
 -        if (isRetriableException(((SolrServerException) err.e).getRootCause())) {
++      if (err.t instanceof SolrServerException) {
++        if (isRetriableException(((SolrServerException) err.t).getRootCause())) {
+           return true;
+         }
+       } else {
 -        if (isRetriableException(err.e)) {
++        if (isRetriableException(err.t)) {
+           return true;
+         }
+       }
        return false;
      }
+     
+     /**
+      * @return true if Solr should retry in case of hitting this exception
+      *         false otherwise
+      */
+     private boolean isRetriableException(Throwable t) {
+       return t instanceof SocketException || t instanceof NoHttpResponseException || t instanceof SocketTimeoutException;
+     }
  
      @Override
      public String getBaseUrl() {
@@@ -448,23 -612,35 +506,35 @@@
      }
  
      @Override
-     public boolean checkRetry() {
-       ZkCoreNodeProps leaderProps;
-       try {
-         leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
-             collection, shardId));
-       } catch (InterruptedException e) {
-         Thread.currentThread().interrupt();
-         return false;
-       } catch (Exception e) {
-         // we retry with same info
-         log.warn(null, e);
-         return true;
+     public boolean checkRetry(Error err) {
+       boolean doRetry = false;
+       if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+         doRetry = true;
        }
-      
-       this.nodeProps = leaderProps;
        
-       return true;
+       // if it's a connect exception, lets try again
 -      if (err.e instanceof SolrServerException && ((SolrServerException) err.e).getRootCause() instanceof ConnectException) {
++      if (err.t instanceof SolrServerException && ((SolrServerException) err.t).getRootCause() instanceof ConnectException) {
+         doRetry = true;
 -      } else if (err.e instanceof ConnectException) {
++      } else if (err.t instanceof ConnectException) {
+         doRetry = true;
+       }
+       if (doRetry) {
+         ZkCoreNodeProps leaderProps;
+         try {
+           leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
+               collection, shardId));
+         } catch (InterruptedException e) {
+           Thread.currentThread().interrupt();
+           return false;
+         } catch (Exception e) {
+           // we retry with same info
+           log.warn(null, e);
+           return true;
+         }
+        
+         this.nodeProps = leaderProps;
+       }
+       return doRetry;
      }
  
      @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
index b843709,0000000..2b8d8a2
mode 100644,000000..100644
--- a/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
+++ b/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
@@@ -1,95 -1,0 +1,154 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.solr.update;
 +
 +import java.io.IOException;
 +import java.net.ConnectException;
 +import java.net.SocketException;
++import java.util.HashSet;
++import java.util.Set;
 +
 +import org.apache.lucene.util.LuceneTestCase;
 +import org.apache.solr.client.solrj.SolrRequest;
 +import org.apache.solr.client.solrj.SolrServerException;
 +import org.apache.solr.client.solrj.impl.Http2SolrClient;
++import org.apache.solr.client.solrj.request.UpdateRequest;
++import org.apache.solr.common.SolrException;
 +import org.apache.solr.common.util.NamedList;
 +
 +public class MockingHttp2SolrClient extends Http2SolrClient {
 +
-   public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION};
++  public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION, BAD_REQUEST};
 +
 +  private volatile Exp exp = null;
++  private boolean oneExpPerReq;
++  private Set<SolrRequest> reqGotException;
 +
 +  public MockingHttp2SolrClient(String baseSolrUrl, Builder builder) {
 +    super(baseSolrUrl, builder);
++    this.oneExpPerReq = builder.oneExpPerReq;
++    this.reqGotException = new HashSet<>();
 +  }
 +
 +  public static class Builder extends Http2SolrClient.Builder {
++    private boolean oneExpPerReq = false;
 +
-     public Builder(String baseSolrUrl, UpdateShardHandlerConfig config) {
-       super(baseSolrUrl);
++    public Builder(UpdateShardHandlerConfig config) {
++      super();
 +      this.connectionTimeout(config.getDistributedConnectionTimeout());
 +      this.idleTimeout(config.getDistributedSocketTimeout());
 +    }
 +
 +    public MockingHttp2SolrClient build() {
-       return new MockingHttp2SolrClient(baseSolrUrl, this);
++      return new MockingHttp2SolrClient(null, this);
++    }
++
++    // DBQ won't cause exception
++    Builder oneExpPerReq() {
++      oneExpPerReq = true;
++      return this;
 +    }
 +  }
 +
 +
 +  public void setExp(Exp exp) {
 +    this.exp = exp;
 +  }
 +
-   private IOException exception() {
++  private Exception exception() {
 +    switch (exp) {
 +      case CONNECT_EXCEPTION:
 +        return new ConnectException();
 +      case SOCKET_EXCEPTION:
 +        return new SocketException();
++      case BAD_REQUEST:
++        return new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request");
 +      default:
 +        break;
 +    }
 +    return null;
 +  }
 +
 +  @Override
 +  public NamedList<Object> request(SolrRequest request, String collection)
 +      throws SolrServerException, IOException {
++    if (request instanceof UpdateRequest) {
++      UpdateRequest ur = (UpdateRequest) request;
++      if (!ur.getDeleteQuery().isEmpty())
++        return super.request(request, collection);
++    }
++
 +    if (exp != null) {
-       if (LuceneTestCase.random().nextBoolean()) {
-         throw exception();
++      if (oneExpPerReq) {
++        if (reqGotException.contains(request))
++          return super.request(request, collection);
++        else
++          reqGotException.add(request);
++      }
++
++      Exception e = exception();
++      if (e instanceof IOException) {
++        if (LuceneTestCase.random().nextBoolean()) {
++          throw (IOException) e;
++        } else {
++          throw new SolrServerException(e);
++        }
++      } else if (e instanceof SolrServerException) {
++        throw (SolrServerException) e;
 +      } else {
-         throw new SolrServerException(exception());
++        throw new SolrServerException(e);
 +      }
 +    }
 +
-     return super.request(request);
++    return super.request(request, collection);
 +  }
 +
-   public Http2ClientResponse request(SolrRequest solrRequest, String collection, OnComplete onComplete)
++  public Http2ClientResponse request(SolrRequest request, String collection, OnComplete onComplete)
 +      throws SolrServerException, IOException {
++    if (request instanceof UpdateRequest) {
++      UpdateRequest ur = (UpdateRequest) request;
++      // won't throw exception if request is DBQ
++      if (ur.getDeleteQuery() != null && !ur.getDeleteQuery().isEmpty())
++        return super.request(request, collection, onComplete);
++    }
++
 +    if (exp != null) {
-       if (LuceneTestCase.random().nextBoolean()) {
-         throw exception();
++      if (oneExpPerReq) {
++        if (reqGotException.contains(request))
++          return super.request(request, collection, onComplete);
++        else
++          reqGotException.add(request);
++      }
++
++      Exception e = exception();
++      if (e instanceof IOException) {
++        if (LuceneTestCase.random().nextBoolean()) {
++          throw (IOException) e;
++        } else {
++          throw new SolrServerException(e);
++        }
++      } else if (e instanceof SolrServerException) {
++        throw (SolrServerException) e;
 +      } else {
-         throw new SolrServerException(exception());
++        throw new SolrServerException(e);
 +      }
 +    }
-     return super.request(solrRequest, collection, onComplete);
++
++    return super.request(request, collection, onComplete);
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
index 551aed5,24cf717..08b8ef9
--- a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
@@@ -16,9 -16,9 +16,10 @@@
   */
  package org.apache.solr.update;
  
 +import javax.xml.parsers.ParserConfigurationException;
  import java.io.File;
  import java.io.IOException;
+ import java.net.SocketException;
  import java.nio.file.Path;
  import java.util.ArrayList;
  import java.util.Collections;
@@@ -45,21 -46,26 +47,29 @@@ import org.apache.solr.core.SolrCore
  import org.apache.solr.core.SolrEventListener;
  import org.apache.solr.index.LogDocMergePolicyFactory;
  import org.apache.solr.search.SolrIndexSearcher;
 -import org.apache.solr.update.MockStreamingSolrClients.Exp;
  import org.apache.solr.update.SolrCmdDistributor.Error;
+ import org.apache.solr.update.SolrCmdDistributor.ForwardNode;
  import org.apache.solr.update.SolrCmdDistributor.Node;
- import org.apache.solr.update.SolrCmdDistributor.RetryNode;
  import org.apache.solr.update.SolrCmdDistributor.StdNode;
  import org.apache.solr.update.processor.DistributedUpdateProcessor;
+ import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;
+ import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
  import org.junit.AfterClass;
  import org.junit.BeforeClass;
  import org.junit.Test;
  import org.xml.sax.SAXException;
  
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.BAD_REQUEST;
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION;
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.SOCKET_EXCEPTION;
++
  // See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
  public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
 -  
 -  private static enum NodeType {FORWARD, STANDARD};
 -  
 +
++  private enum NodeType {FORWARD, STANDARD};
++
    private AtomicInteger id = new AtomicInteger();
 -  
 +
    @BeforeClass
    public static void beforeClass() throws Exception {
      // we can't use the Randomized merge policy because the test depends on
@@@ -323,33 -330,178 +334,181 @@@
              ((NamedList<Object>) resp.get("index")).get("maxDoc"));
        }
      }
 -    
 +
-     testMaxRetries();
-     testOneRetry();
+     testMaxRetries(NodeType.FORWARD);
+     testMaxRetries(NodeType.STANDARD);
+     testOneRetry(NodeType.FORWARD);
+     testOneRetry(NodeType.STANDARD);
      testRetryNodeAgainstBadAddress();
-     testRetryNodeWontRetrySocketError();
- 
+     testStdNodeRetriesSocketError();
+     testForwardNodeWontRetrySocketError();
+     testNodeWontRetryBadRequest(NodeType.FORWARD);
+     testNodeWontRetryBadRequest(NodeType.STANDARD);
+     testMinRfOnRetries(NodeType.FORWARD);
+     testMinRfOnRetries(NodeType.STANDARD);
      testDistribOpenSearcher();
+     testReqShouldRetryNoRetries();
+     testReqShouldRetryMaxRetries();
+     testReqShouldRetryBadRequest();
+     testReqShouldRetryNotFound();
+     testReqShouldRetryDBQ();
+     testDeletes(false, true);
+     testDeletes(false, false);
+     testDeletes(true, true);
+     testDeletes(true, false);
    }
 -  
 +
-   private void testMaxRetries() throws IOException {
-     final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
-     try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
-       SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
-       solrClient.setExp(MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION);
+   private void testDeletes(boolean dbq, boolean withFailures) throws Exception {
+     final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+     solrclient.commit(true, true);
+     long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+         .getNumFound();
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
++    final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT)
++        .oneExpPerReq();
++    try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
+       if (withFailures) {
 -        streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++        streamingClients.setExp(CONNECT_EXCEPTION);
+       }
        ArrayList<Node> nodes = new ArrayList<>();
-       final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
  
 -      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
 -          ZkStateReader.CORE_NAME_PROP, "");
 -
        final AtomicInteger retries = new AtomicInteger();
-       ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
-       RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
 -      nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
          @Override
-         public boolean checkRetry() {
+         public boolean checkRetry(Error err) {
 -          streamingClients.setExp(null);
            retries.incrementAndGet();
-           return true;
+           return super.checkRetry(err);
          }
        };
  
+ 
+       nodes.add(retryNode);
+ 
+       for (int i = 0 ; i < 5 ; i++) {
+         AddUpdateCommand cmd = new AddUpdateCommand(null);
+         int currentId = id.incrementAndGet();
+         cmd.solrDoc = sdoc("id", currentId);
+         ModifiableSolrParams params = new ModifiableSolrParams();
+         cmdDistrib.distribAdd(cmd, nodes, params);
++        cmdDistrib.blockUntilFinished();
++
+         DeleteUpdateCommand dcmd = new DeleteUpdateCommand(null);
+         if (dbq) {
+           dcmd.setQuery("id:" + currentId);
+         } else {
+           dcmd.setId(String.valueOf(currentId));
+         }
+         cmdDistrib.distribDelete(dcmd, nodes, params, false, null, null);
+       }
 -      
+ 
++
++      streamingClients.setExp(null);
+       CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+       cmdDistrib.distribCommit(ccmd, nodes, new ModifiableSolrParams());
+       cmdDistrib.finish();
 -      
++
+       int expectedRetryCount = 0;
+       if (withFailures) {
+         if (dbq) {
 -          expectedRetryCount = 1; // just the first cmd would be retried
++          expectedRetryCount = 5;
+         } else {
+           expectedRetryCount = 10;
+         }
+       }
+       assertEquals(expectedRetryCount, retries.get());
+ 
+ 
+       long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+           .getNumFound();
+ 
+       // we will get java.net.ConnectException which we retry on
+       assertEquals(numFoundBefore, numFoundAfter);
+       assertEquals(0, cmdDistrib.getErrors().size());
+     }
+   }
+ 
+   private void testMinRfOnRetries(NodeType nodeType) throws Exception {
+     final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++    final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT);
++    try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(CONNECT_EXCEPTION);
+       ArrayList<Node> nodes = new ArrayList<>();
+ 
+       ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+           ZkStateReader.CORE_NAME_PROP, "");
+ 
+       final AtomicInteger retries = new AtomicInteger();
+       nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       if (nodeType == NodeType.FORWARD) {
+         nodes.add(new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             if (retries.incrementAndGet() >= 3) {
+               streamingClients.setExp(null);
+             }
+             return super.checkRetry(err);
+           }
+         });
+       } else {
+         nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             if (retries.incrementAndGet() >= 3) {
+               streamingClients.setExp(null);
+             }
+             return super.checkRetry(err);
+           }
+         });
+       }
+ 
+ 
+       AddUpdateCommand cmd = new AddUpdateCommand(null);
+       cmd.solrDoc = sdoc("id", id.incrementAndGet());
+       ModifiableSolrParams params = new ModifiableSolrParams();
+       RollupRequestReplicationTracker rollupReqTracker = new RollupRequestReplicationTracker("2");
+       LeaderRequestReplicationTracker leaderReqTracker = new LeaderRequestReplicationTracker("shard1", 2);
+ 
+       cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReqTracker, leaderReqTracker);
+       cmdDistrib.finish();
+       assertEquals(3, retries.get());
+       assertEquals(2, leaderReqTracker.getAchievedRf());// "2" here is because one would be the leader, that creates the instance of LeaderRequestReplicationTracker, the second one is the node
+ 
+       assertEquals(0, cmdDistrib.getErrors().size());
+     }
+   }
+ 
+   private void testMaxRetries(NodeType nodeType) throws IOException {
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++    final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT);
++    try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(CONNECT_EXCEPTION);
+       ArrayList<Node> nodes = new ArrayList<>();
+       final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
+ 
+       final AtomicInteger retries = new AtomicInteger();
+       ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       Node retryNode;
+       if (nodeType == NodeType.FORWARD) {
+         retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 6) {
+           @Override
+           public boolean checkRetry(Error err) {
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       } else {
+         retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 6) {
+           @Override
+           public boolean checkRetry(Error err) {
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       }
 -      
++
+ 
        nodes.add(retryNode);
  
        AddUpdateCommand cmd = new AddUpdateCommand(null);
@@@ -364,30 -516,84 +523,81 @@@
        assertEquals(1, cmdDistrib.getErrors().size());
      }
    }
 -  
 +
-   private void testOneRetry() throws Exception {
+   private void testReqShouldRetryNoRetries() {
 -    Error err = getError(new SocketException()); 
++    Error err = getError(new SocketException());
+     SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 0), new UpdateRequest(), true);
+     assertFalse(req.shouldRetry(err));
+   }
 -  
++
+   private void testReqShouldRetryDBQ() {
 -    Error err = getError(new SocketException()); 
++    Error err = getError(new SocketException());
+     UpdateRequest dbqReq = new UpdateRequest();
+     dbqReq.deleteByQuery("*:*");
+     SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), dbqReq, true);
+     assertFalse(req.shouldRetry(err));
+   }
 -  
++
+   private void testReqShouldRetryMaxRetries() {
 -    Error err = getError(new SocketException()); 
++    Error err = getError(new SocketException());
+     SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+     assertTrue(req.shouldRetry(err));
+     req.retries++;
+     assertFalse(req.shouldRetry(err));
+   }
 -  
++
+   private void testReqShouldRetryBadRequest() {
 -    Error err = getError(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "bad request")); 
++    Error err = getError(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "bad request"));
+     SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+     assertFalse(req.shouldRetry(err));
+   }
 -  
++
+   private void testReqShouldRetryNotFound() {
+     Error err = getError(new SolrException(SolrException.ErrorCode.NOT_FOUND, "not found"));
+     SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+     assertTrue(req.shouldRetry(err));
+   }
 -  
++
+   private Error getError(Exception e) {
+     Error err = new Error();
 -    err.e = e;
++    err.t = e;
+     if (e instanceof SolrException) {
+       err.statusCode = ((SolrException)e).code();
+     }
+     return err;
+   }
 -  
++
+   private void testOneRetry(NodeType nodeType) throws Exception {
      final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
      long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
          .getNumFound();
-     final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
-     try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
-       SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
-       solrClient.setExp(MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION);
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++    try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(CONNECT_EXCEPTION);
        ArrayList<Node> nodes = new ArrayList<>();
  
--      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
--          ZkStateReader.CORE_NAME_PROP, "");
--
        final AtomicInteger retries = new AtomicInteger();
--      nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
-       RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
-         @Override
-         public boolean checkRetry() {
-           solrClient.setExp(null);
-           retries.incrementAndGet();
-           return true;
-         }
-       };
++      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       Node retryNode;
+       if (nodeType == NodeType.FORWARD) {
+         retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             streamingClients.setExp(null);
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       } else {
+         retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             streamingClients.setExp(null);
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       }
  
  
        nodes.add(retryNode);
@@@ -417,22 -624,78 +628,75 @@@
      final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
      long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
          .getNumFound();
-     final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
-     try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
-       SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
-       solrClient.setExp(MockingHttp2SolrClient.Exp.SOCKET_EXCEPTION);
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.BAD_REQUEST);
++    try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(BAD_REQUEST);
        ArrayList<Node> nodes = new ArrayList<>();
- 
        ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
            ZkStateReader.CORE_NAME_PROP, "");
  
        final AtomicInteger retries = new AtomicInteger();
-       nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
-       RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
+       Node retryNode;
+       if (nodeType == NodeType.FORWARD) {
+         retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       } else {
+         retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+           @Override
+           public boolean checkRetry(Error err) {
+             retries.incrementAndGet();
+             return super.checkRetry(err);
+           }
+         };
+       }
+       nodes.add(retryNode);
+ 
+       AddUpdateCommand cmd = new AddUpdateCommand(null);
+       cmd.solrDoc = sdoc("id", id.incrementAndGet());
+       ModifiableSolrParams params = new ModifiableSolrParams();
+ 
+       CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+       cmdDistrib.distribAdd(cmd, nodes, params);
+ 
+       streamingClients.setExp(null);
+       cmdDistrib.distribCommit(ccmd, nodes, params);
+       cmdDistrib.finish();
+ 
+       // it will checkRetry, but not actually do it...
+       assertEquals(1, retries.get());
+ 
+ 
+       long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+           .getNumFound();
+ 
+       // we will get java.net.SocketException: Network is unreachable, which we don't retry on
+       assertEquals(numFoundBefore, numFoundAfter);
+       assertEquals(1, cmdDistrib.getErrors().size());
+       unIgnoreException("Bad Request");
+     }
+   }
 -  
++
+   private void testForwardNodeWontRetrySocketError() throws Exception {
+     final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+     long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+         .getNumFound();
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.SOCKET_EXCEPTION);
++    try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(SOCKET_EXCEPTION);
+       ArrayList<Node> nodes = new ArrayList<>();
+ 
 -      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
 -          ZkStateReader.CORE_NAME_PROP, "");
 -
+       final AtomicInteger retries = new AtomicInteger();
 -      nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
          @Override
-         public boolean checkRetry() {
+         public boolean checkRetry(Error err) {
            retries.incrementAndGet();
-           return true;
+           return super.checkRetry(err);
          }
        };
  
@@@ -462,26 -725,24 +726,21 @@@
        assertEquals(1, cmdDistrib.getErrors().size());
      }
    }
 -  
 +
-   private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
-     // Test RetryNode
-     {
-       SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
-       final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
-       long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
-           .getNumFound();
- 
+   private void testStdNodeRetriesSocketError() throws Exception {
+     final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
 -    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
 -      streamingClients.setExp(Exp.SOCKET_EXCEPTION);
++    try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++      SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++      streamingClients.setExp(SOCKET_EXCEPTION);
        ArrayList<Node> nodes = new ArrayList<>();
  
-       ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
-       RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
 -      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
 -          ZkStateReader.CORE_NAME_PROP, "");
 -
+       final AtomicInteger retries = new AtomicInteger();
 -      nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+       Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
          @Override
-         public boolean checkRetry() {
-           ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
-               ZkStateReader.CORE_NAME_PROP, "");
-           this.nodeProps = new ZkCoreNodeProps(leaderProps);
- 
-           return true;
+         public boolean checkRetry(Error err) {
+           retries.incrementAndGet();
+           return super.checkRetry(err);
          }
        };
  
@@@ -494,29 -754,66 +752,65 @@@
        ModifiableSolrParams params = new ModifiableSolrParams();
  
        cmdDistrib.distribAdd(cmd, nodes, params);
- 
-       CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
-       params = new ModifiableSolrParams();
-       params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
-       cmdDistrib.distribCommit(ccmd, nodes, params);
        cmdDistrib.finish();
  
-       long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
-           .getNumFound();
+       // it will checkRetry, but not actually do it...
+       assertEquals(6, retries.get());
+     }
+   }
  
-       // different OS's will throw different exceptions for the bad address above
-       if (numFoundBefore != numFoundAfter) {
-         assertEquals(0, cmdDistrib.getErrors().size());
-         assertEquals(numFoundBefore + 1, numFoundAfter);
-       } else {
-         // we will get java.net.SocketException: Network is unreachable and not retry
-         assertEquals(numFoundBefore, numFoundAfter);
+   private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
+     // Test RetryNode
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler)) {
 -      final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
 -      long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
 -          .getNumFound();
++    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
++    final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
++    long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
++        .getNumFound();
+ 
 -      ArrayList<Node> nodes = new ArrayList<>();
++    ArrayList<Node> nodes = new ArrayList<>();
  
-         assertEquals(1, cmdDistrib.getErrors().size());
 -      ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
 -      ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
 -        @Override
 -        public boolean checkRetry(Error err) {
 -          ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
 -              ZkStateReader.CORE_NAME_PROP, "");
 -          this.nodeProps = new ZkCoreNodeProps(leaderProps);
++    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
++    ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
++      @Override
++      public boolean checkRetry(Error err) {
++        ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
++            ZkStateReader.CORE_NAME_PROP, "");
++        this.nodeProps = new ZkCoreNodeProps(leaderProps);
+ 
 -          return super.checkRetry(err);
 -        }
 -      };
++        return super.checkRetry(err);
 +      }
++    };
+ 
+ 
 -      nodes.add(retryNode);
++    nodes.add(retryNode);
+ 
+ 
 -      AddUpdateCommand cmd = new AddUpdateCommand(null);
 -      cmd.solrDoc = sdoc("id", id.incrementAndGet());
 -      ModifiableSolrParams params = new ModifiableSolrParams();
++    AddUpdateCommand cmd = new AddUpdateCommand(null);
++    cmd.solrDoc = sdoc("id", id.incrementAndGet());
++    ModifiableSolrParams params = new ModifiableSolrParams();
+ 
 -      cmdDistrib.distribAdd(cmd, nodes, params);
++    cmdDistrib.distribAdd(cmd, nodes, params);
+ 
 -      CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
 -      params = new ModifiableSolrParams();
 -      params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
 -      cmdDistrib.distribCommit(ccmd, nodes, params);
 -      cmdDistrib.finish();
++    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
++    params = new ModifiableSolrParams();
++    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
++    cmdDistrib.distribCommit(ccmd, nodes, params);
++    cmdDistrib.finish();
+ 
 -      long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
 -          .getNumFound();
++    long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
++        .getNumFound();
+ 
 -      // different OS's will throw different exceptions for the bad address above
 -      if (numFoundBefore != numFoundAfter) {
 -        assertEquals(0, cmdDistrib.getErrors().size());
 -        assertEquals(numFoundBefore + 1, numFoundAfter);
 -      } else {
 -        // we will get java.net.SocketException: Network is unreachable and not retry
 -        assertEquals(numFoundBefore, numFoundAfter);
++    // different OS's will throw different exceptions for the bad address above
++    if (numFoundBefore != numFoundAfter) {
++      assertEquals(0, cmdDistrib.getErrors().size());
++      assertEquals(numFoundBefore + 1, numFoundAfter);
++    } else {
++      // we will get java.net.SocketException: Network is unreachable and not retry
++      assertEquals(numFoundBefore, numFoundAfter);
+ 
 -        assertEquals(1, cmdDistrib.getErrors().size());
 -      }
++      assertEquals(1, cmdDistrib.getErrors().size());
      }
    }
 -  
 +
    @Override
    public void distribTearDown() throws Exception {
      updateShardHandler.close();
@@@ -524,23 -821,22 +818,21 @@@
    }
  
    private void testDistribOpenSearcher() {
-     {
-       SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
 -    try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler)) {
--      UpdateRequest updateRequest = new UpdateRequest();
++    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
++    UpdateRequest updateRequest = new UpdateRequest();
  
--      CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
++    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
  
--      //test default value (should be true)
--      cmdDistrib.addCommit(updateRequest, ccmd);
--      boolean openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, false);
--      assertTrue(openSearcher);
++    //test default value (should be true)
++    cmdDistrib.addCommit(updateRequest, ccmd);
++    boolean openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, false);
++    assertTrue(openSearcher);
  
--      //test openSearcher = false
--      ccmd.openSearcher = false;
++    //test openSearcher = false
++    ccmd.openSearcher = false;
  
--      cmdDistrib.addCommit(updateRequest, ccmd);
--      openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, true);
--      assertFalse(openSearcher);
--    }
++    cmdDistrib.addCommit(updateRequest, ccmd);
++    openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, true);
++    assertFalse(openSearcher);
    }
  }


Mime
View raw message