lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject [15/18] lucene-solr:feature/autoscaling: Merge branch 'master' into feature/autoscaling
Date Wed, 28 Jun 2017 06:51:54 GMT
Merge branch 'master' into feature/autoscaling

# Conflicts:
#	solr/CHANGES.txt
#	solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
#	solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
#	solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4239896e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4239896e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4239896e

Branch: refs/heads/feature/autoscaling
Commit: 4239896e6dfeb9ef1b41313e025b5e2f90675817
Parents: 257ba05 13a3ae2
Author: Shalin Shekhar Mangar <shalin@apache.org>
Authored: Wed Jun 28 11:09:19 2017 +0530
Committer: Shalin Shekhar Mangar <shalin@apache.org>
Committed: Wed Jun 28 11:09:19 2017 +0530

----------------------------------------------------------------------
 .../lucene/spatial-extras/spatial-extras.iml    |   5 +-
 lucene/CHANGES.txt                              |   8 +
 .../byTask/feeds/SpatialFileQueryMaker.java     |  15 +-
 .../org/apache/lucene/analysis/Analyzer.java    |   4 +-
 .../org/apache/lucene/search/DoubleValues.java  |  21 +
 .../lucene/search/DoubleValuesSource.java       | 216 ++++----
 .../apache/lucene/search/LongValuesSource.java  |  82 ++-
 .../lucene/search/TestDoubleValuesSource.java   |  16 +-
 .../facet/range/TestRangeFacetCounts.java       |  75 +--
 .../queries/function/FunctionScoreQuery.java    |   9 -
 .../lucene/queries/function/ValueSource.java    | 260 +++++++---
 .../function/TestFunctionScoreExplanations.java |   8 +-
 .../function/TestFunctionScoreQuery.java        |  91 +++-
 lucene/spatial-extras/build.xml                 |   8 +-
 .../org/apache/lucene/spatial/ShapeValues.java  |  41 ++
 .../lucene/spatial/ShapeValuesSource.java       |  34 ++
 .../apache/lucene/spatial/SpatialStrategy.java  |  19 +-
 .../bbox/BBoxOverlapRatioValueSource.java       |   7 +-
 .../spatial/bbox/BBoxSimilarityValueSource.java |  79 ++-
 .../lucene/spatial/bbox/BBoxStrategy.java       |  13 +-
 .../lucene/spatial/bbox/BBoxValueSource.java    |  74 +--
 .../composite/CompositeSpatialStrategy.java     |  14 +-
 .../spatial/composite/CompositeVerifyQuery.java |  32 +-
 .../composite/IntersectsRPTVerifyQuery.java     |  21 +-
 .../prefix/NumberRangePrefixTreeStrategy.java   |   4 +-
 .../spatial/prefix/PrefixTreeStrategy.java      |   4 +-
 .../serialized/SerializedDVStrategy.java        | 110 +---
 .../spatial/util/CachingDoubleValueSource.java  |  61 ++-
 .../util/DistanceToShapeValueSource.java        |  68 +--
 .../util/ReciprocalDoubleValuesSource.java      |  96 ++++
 .../spatial/util/ShapeAreaValueSource.java      |  67 +--
 .../ShapeFieldCacheDistanceValueSource.java     |  59 ++-
 .../spatial/util/ShapePredicateValueSource.java | 113 -----
 .../spatial/util/ShapeValuesPredicate.java      |  99 ++++
 .../spatial/vector/DistanceValueSource.java     |  72 +--
 .../spatial/vector/PointVectorStrategy.java     |  95 +++-
 .../lucene/spatial/DistanceStrategyTest.java    |   6 -
 .../apache/lucene/spatial/SpatialExample.java   |  12 +-
 .../apache/lucene/spatial/StrategyTestCase.java |  39 +-
 .../lucene/spatial/spatial4j/Geo3dRptTest.java  |   2 +-
 .../Geo3dShapeRectRelationTestCase.java         |   2 +-
 .../DocumentValueSourceDictionaryTest.java      |  15 +
 solr/CHANGES.txt                                |  10 +-
 solr/bin/solr                                   |  13 +-
 .../ltr/model/MultipleAdditiveTreesModel.java   |  16 +-
 .../src/java/org/apache/solr/cloud/Assign.java  |   6 +-
 .../cloud/OverseerCollectionMessageHandler.java |   5 +-
 .../org/apache/solr/cloud/ZkController.java     |  19 +-
 .../cloud/autoscaling/AutoScalingHandler.java   |  13 +-
 .../org/apache/solr/core/CoreContainer.java     |   2 +-
 .../solr/handler/admin/CollectionsHandler.java  |   2 +-
 .../org/apache/solr/legacy/BBoxStrategy.java    |  13 +-
 .../org/apache/solr/legacy/BBoxValueSource.java |  82 +--
 .../apache/solr/legacy/DistanceValueSource.java |  81 ++-
 .../apache/solr/legacy/PointVectorStrategy.java |  17 +-
 .../transform/GeoTransformerFactory.java        |  33 +-
 .../solr/schema/AbstractSpatialFieldType.java   |  10 +-
 .../java/org/apache/solr/schema/BBoxField.java  |   8 +-
 .../solr/schema/LatLonPointSpatialField.java    |  46 +-
 .../schema/RptWithGeometrySpatialField.java     |  60 +--
 .../apache/solr/schema/ZkIndexSchemaReader.java | 105 ++--
 .../distance/GeoDistValueSourceParser.java      |   8 +-
 .../configsets/_default/conf/solrconfig.xml     |   2 +-
 .../solr/cloud/CollectionsAPISolrJTest.java     |   2 +-
 .../apache/solr/schema/SchemaWatcherTest.java   |  56 +++
 .../apache/solr/search/TestSolr4Spatial.java    |  10 +-
 solr/solr-ref-guide/src/graph-traversal.adoc    | 296 +++++------
 solr/solr-ref-guide/src/stream-sources.adoc     |   4 +-
 .../cloud/autoscaling/AddReplicaSuggester.java  |  73 +++
 .../client/solrj/cloud/autoscaling/Cell.java    |  69 +++
 .../client/solrj/cloud/autoscaling/Clause.java  | 480 ++++++++++++++++++
 .../cloud/autoscaling/ClusterDataProvider.java  |  52 ++
 .../cloud/autoscaling/MoveReplicaSuggester.java |  83 +++
 .../client/solrj/cloud/autoscaling/Operand.java | 124 +++++
 .../client/solrj/cloud/autoscaling/Policy.java  | 499 +++++++++++++++++++
 .../solrj/cloud/autoscaling/PolicyHelper.java   |  96 ++++
 .../solrj/cloud/autoscaling/Preference.java     |  92 ++++
 .../client/solrj/cloud/autoscaling/Row.java     | 140 ++++++
 .../solrj/cloud/autoscaling/package-info.java   |  23 +
 .../solrj/impl/SolrClientDataProvider.java      |   4 +-
 .../solrj/request/CollectionAdminRequest.java   |   2 +-
 .../cloud/autoscaling/AddReplicaSuggester.java  |  73 ---
 .../org/apache/solr/cloud/autoscaling/Cell.java |  57 ---
 .../apache/solr/cloud/autoscaling/Clause.java   | 467 -----------------
 .../cloud/autoscaling/ClusterDataProvider.java  |  52 --
 .../cloud/autoscaling/MoveReplicaSuggester.java |  83 ---
 .../apache/solr/cloud/autoscaling/Operand.java  | 124 -----
 .../apache/solr/cloud/autoscaling/Policy.java   | 497 ------------------
 .../solr/cloud/autoscaling/PolicyHelper.java    |  96 ----
 .../solr/cloud/autoscaling/Preference.java      |  88 ----
 .../org/apache/solr/cloud/autoscaling/Row.java  | 135 -----
 .../solr/cloud/autoscaling/package-info.java    |  23 -
 .../apache/solr/common/cloud/DocCollection.java |   2 +-
 .../solr/cloud/autoscaling/TestPolicy.java      |  20 +-
 94 files changed, 3426 insertions(+), 3023 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index d614519,3f25c26..d8c90a2
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -224,13 -211,9 +224,16 @@@ Bug Fixe
  
  * SOLR-10948: Fix extraction component to treat DatePointField the same as TrieDateField (hossman)
  
+ * SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader.
+   (Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
+ 
 +* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
 +
 +* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
 +  config watcher is fired. (shalin)
 +
 +* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
 +
  Optimizations
  ----------------------
  

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 5e16b5d,c6051d7..3ee1fde
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@@ -42,9 -43,6 +43,9 @@@ import org.apache.solr.client.solrj.imp
  import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
  import org.apache.solr.client.solrj.request.UpdateRequest;
  import org.apache.solr.client.solrj.response.UpdateResponse;
 +import org.apache.solr.cloud.autoscaling.AutoScaling;
 +import org.apache.solr.cloud.autoscaling.AutoScalingHandler;
- import org.apache.solr.cloud.autoscaling.Policy;
++import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
  import org.apache.solr.cloud.overseer.OverseerAction;
  import org.apache.solr.cloud.rule.ReplicaAssigner;
  import org.apache.solr.cloud.rule.ReplicaAssigner.Position;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index 0000000,01149f3..7b00c87
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@@ -1,0 -1,69 +1,73 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.util.List;
+ 
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
++import org.apache.solr.common.cloud.Replica;
+ 
+ class AddReplicaSuggester extends Suggester {
+ 
+   SolrRequest init() {
+     SolrRequest operation = tryEachNode(true);
+     if (operation == null) operation = tryEachNode(false);
+     return operation;
+   }
+ 
+   SolrRequest tryEachNode(boolean strict) {
+     String coll = (String) hints.get(Hint.COLL);
+     String shard = (String) hints.get(Hint.SHARD);
++    Replica.Type type = Replica.Type.get((String) hints.get(Hint.REPLICATYPE));
+     if (coll == null || shard == null)
+       throw new RuntimeException("add-replica requires 'collection' and 'shard'");
+     //iterate through elements and identify the least loaded
+ 
+     List<Clause.Violation> leastSeriousViolation = null;
+     Integer targetNodeIndex = null;
+     for (int i = getMatrix().size() - 1; i >= 0; i--) {
+       Row row = getMatrix().get(i);
++      if(!row.isLive) continue;
+       if (!isAllowed(row.node, Hint.TARGET_NODE)) continue;
 -      Row tmpRow = row.addReplica(coll, shard);
++      Row tmpRow = row.addReplica(coll, shard, type);
+       tmpRow.violations.clear();
+ 
+       List<Clause.Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getMatrix(), tmpRow, i));
+       if(!containsNewErrors(errs)) {
+         if(isLessSerious(errs, leastSeriousViolation)){
+           leastSeriousViolation = errs;
+           targetNodeIndex = i;
+         }
+       }
+     }
+ 
+     if (targetNodeIndex != null) {// there are no rule violations
 -      getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard));
++      getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(coll, shard, type));
+       return CollectionAdminRequest
+           .addReplicaToShard(coll, shard)
++          .setType(type)
+           .setNode(getMatrix().get(targetNodeIndex).node);
+     }
+ 
+     return null;
+   }
+ 
+ 
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 0000000,ff56627..8a5a121
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@@ -1,0 -1,472 +1,480 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.HashSet;
++import java.util.LinkedHashMap;
+ import java.util.List;
++import java.util.Locale;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.Optional;
+ import java.util.Set;
 -import java.util.concurrent.atomic.AtomicInteger;
+ 
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.common.MapWriter;
++import org.apache.solr.common.cloud.Replica;
+ import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+ 
+ import static java.util.Collections.singletonMap;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+ import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
+ import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+ import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+ 
+ // a set of conditions in a policy
+ public class Clause implements MapWriter, Comparable<Clause> {
+   public Map<String, Object> original;
+   public Condition collection, shard, replica, tag, globalTag;
++  public final Replica.Type type;
+ 
+   boolean strict = true;
+ 
+   public Clause(Map<String, Object> m) {
+     this.original = m;
++    String type = (String) m.get("type");
++    this.type = type == null || ANY.equals(type) ? null : Replica.Type.valueOf(type.toUpperCase(Locale.ROOT));
+     strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true")));
+     Optional<String> globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst();
+     if (globalTagName.isPresent()) {
+       globalTag = parse(globalTagName.get(), m);
+       if (m.size() > 2) {
+         throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
+       }
+       tag = parse(m.keySet().stream()
+           .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
+           .findFirst().get(), m);
+     } else {
+       collection = parse(COLLECTION, m);
+       shard = parse(SHARD, m);
+       if(m.get(REPLICA) == null){
+         throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
+       }
+       this.replica = parse(REPLICA, m);
+       if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
+       m.forEach((s, o) -> parseCondition(s, o));
+     }
+     if (tag == null)
+       throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+ 
+   }
+ 
+   public boolean doesOverride(Clause that) {
+     return (collection.equals(that.collection) &&
+         tag.name.equals(that.tag.name));
+ 
+   }
+ 
+   public boolean isPerCollectiontag() {
+     return globalTag == null;
+   }
+ 
+   void parseCondition(String s, Object o) {
+     if (IGNORE_TAGS.contains(s)) return;
+     if (tag != null) {
+       throw new IllegalArgumentException("Only one tag other than collection, shard, replica is possible");
+     }
+     tag = parse(s, singletonMap(s, o));
+   }
+ 
++  private int compareTypes(Replica.Type t1, Replica.Type t2) {
++    if (t1 == null && t2 == null) return 0;
++    if (t1 != null && t2 == null) return -1;
++    if (t1 == null) return 1;
++    return 0;
++  }
++
+   @Override
+   public int compareTo(Clause that) {
 -    try {
 -      int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
 -      if (v != 0) return v;
 -      if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
 -        v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
 -        if (v == 0) {
 -          v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
 -          v = this.replica.op == LESS_THAN ? v : v * -1;
 -        }
 -        return v;
 -      } else {
 -        return 0;
++    int v = Integer.compare(this.tag.op.priority, that.tag.op.priority);
++    if (v != 0) return v;
++    if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
++      v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
++      if (v == 0) {// higher the number of replicas , harder to satisfy
++        v = Long.compare((Long) this.replica.val, (Long) that.replica.val);
++        v = this.replica.op == LESS_THAN ? v : v * -1;
+       }
 -    } catch (NullPointerException e) {
 -      throw e;
++      if (v == 0) v = compareTypes(this.type, that.type);
++      return v;
++    } else {
++      return 0;
+     }
++
+   }
+ 
+   void addTags(List<String> params) {
+     if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name);
+     if (tag != null && !params.contains(tag.name)) params.add(tag.name);
+   }
+ 
+   public static class Condition {
+     final String name;
+     final Object val;
+     final Operand op;
+ 
+     Condition(String name, Object val, Operand op) {
+       this.name = name;
+       this.val = val;
+       this.op = op;
+     }
+ 
 -    TestStatus match(Row row) {
 -      return op.match(val, row.getVal(name));
 -    }
 -
 -    TestStatus match(Object testVal) {
 -      return op.match(this.val, testVal);
 -    }
 -
+     public boolean isPass(Object inputVal) {
++      if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(type);
+       return op.match(val, validate(name, inputVal, false)) == PASS;
+     }
+ 
+     public boolean isPass(Row row) {
+       return op.match(val, row.getVal(name)) == PASS;
+     }
+ 
+     @Override
+     public boolean equals(Object that) {
+       if (that instanceof Condition) {
+         Condition c = (Condition) that;
+         return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op;
+       }
+       return false;
+     }
+ 
 -    public Integer delta(Object val) {
++    public Long delta(Object val) {
+       return op.delta(this.val, val);
+     }
+ 
+     public String getName() {
+       return name;
+     }
+ 
+     public Object getValue() {
+       return val;
+     }
 -    
++
+     public Operand getOperand() {
+       return op;
+     }
+   }
+ 
+   static Condition parse(String s, Map m) {
+     Object expectedVal = null;
+     Object val = m.get(s);
+     try {
+       String conditionName = s.trim();
+       Operand operand = null;
+       if (val == null) {
+         operand = WILDCARD;
+         expectedVal = Policy.ANY;
+       } else if (val instanceof String) {
+         String strVal = ((String) val).trim();
+         if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
+         else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
+         else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
+         else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
+         else operand = EQUAL;
+         expectedVal = validate(s, strVal.substring(EQUAL == operand || WILDCARD == operand ? 0 : 1), true);
+       } else if (val instanceof Number) {
+         operand = EQUAL;
+         expectedVal = validate(s, val, true);
+       }
+       return new Condition(conditionName, expectedVal, operand);
+ 
+     } catch (Exception e) {
+       throw new IllegalArgumentException("Invalid tag : " + s + ":" + val, e);
+     }
+   }
+ 
+   public class Violation implements MapWriter {
+     final String shard, coll, node;
+     final Object actualVal;
 -    final Integer delta;//how far is the actual value from the expected value
++    final Long delta;//how far is the actual value from the expected value
+     final Object tagKey;
+     private final int hash;
+ 
+ 
 -    private Violation(String coll, String shard, String node, Object actualVal, Integer delta, Object tagKey) {
++    private Violation(String coll, String shard, String node, Object actualVal, Long delta, Object tagKey) {
+       this.shard = shard;
+       this.coll = coll;
+       this.node = node;
+       this.delta = delta;
+       this.actualVal = actualVal;
+       this.tagKey = tagKey;
+       hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode();
+     }
+ 
+     public Clause getClause() {
+       return Clause.this;
+     }
+ 
+     @Override
+     public int hashCode() {
+       return hash;
+     }
+     //if the delta is lower , this violation is less serious
+     public boolean isLessSerious(Violation that) {
+       return that.delta != null && delta != null &&
+           Math.abs(delta) < Math.abs(that.delta);
+     }
+ 
+     @Override
+     public boolean equals(Object that) {
+       if (that instanceof Violation) {
+         Violation v = (Violation) that;
+         return Objects.equals(this.shard, v.shard) &&
+             Objects.equals(this.coll, v.coll) &&
+             Objects.equals(this.node, v.node) &&
+             Objects.equals(this.tagKey, v.tagKey)
+             ;
+       }
+       return false;
+     }
+ 
+     @Override
++    public String toString() {
++      return Utils.toJSONString(Utils.getDeepCopy(toMap(new LinkedHashMap<>()), 5));
++    }
++
++    @Override
+     public void writeMap(EntryWriter ew) throws IOException {
+       ew.putIfNotNull("collection", coll);
+       ew.putIfNotNull("shard", shard);
+       ew.putIfNotNull("node", node);
+       ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+       ew.putIfNotNull("violation", (MapWriter) ew1 -> {
 -        ew1.put(getClause().isPerCollectiontag() ? "replica" : tag.name,
 -            String.valueOf(actualVal));
++        if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
++        else ew1.put(tag.name, String.valueOf(actualVal));
+         ew1.putIfNotNull("delta", delta);
+       });
+       ew.put("clause", getClause());
+     }
+   }
+ 
+ 
+   public List<Violation> test(List<Row> allRows) {
+     List<Violation> violations = new ArrayList<>();
+     if (isPerCollectiontag()) {
 -      Map<String, Map<String, Map<String, AtomicInteger>>> replicaCount = computeReplicaCounts(allRows);
 -      for (Map.Entry<String, Map<String, Map<String, AtomicInteger>>> e : replicaCount.entrySet()) {
++      Map<String, Map<String, Map<String, ReplicaCount>>> replicaCount = computeReplicaCounts(allRows);
++      for (Map.Entry<String, Map<String, Map<String, ReplicaCount>>> e : replicaCount.entrySet()) {
+         if (!collection.isPass(e.getKey())) continue;
 -        for (Map.Entry<String, Map<String, AtomicInteger>> shardVsCount : e.getValue().entrySet()) {
++        for (Map.Entry<String, Map<String, ReplicaCount>> shardVsCount : e.getValue().entrySet()) {
+           if (!shard.isPass(shardVsCount.getKey())) continue;
 -          for (Map.Entry<String, AtomicInteger> counts : shardVsCount.getValue().entrySet()) {
++          for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
+             if (!replica.isPass(counts.getValue())) {
+               violations.add(new Violation(
+                   e.getKey(),
+                   shardVsCount.getKey(),
+                   tag.name.equals("node") ? counts.getKey() : null,
+                   counts.getValue(),
+                   replica.delta(counts.getValue()),
+                   counts.getKey()
+               ));
+             }
+           }
+         }
+       }
+     } else {
+       for (Row r : allRows) {
+         if (!tag.isPass(r)) {
+           violations.add(new Violation(null, null, r.node, r.getVal(tag.name), tag.delta(r.getVal(tag.name)), null));
+         }
+       }
+     }
+     return violations;
+ 
+   }
+ 
+ 
 -  private Map<String, Map<String, Map<String, AtomicInteger>>> computeReplicaCounts(List<Row> allRows) {
 -    Map<String, Map<String, Map<String, AtomicInteger>>> collVsShardVsTagVsCount = new HashMap<>();
 -    for (Row row : allRows)
++  private Map<String, Map<String, Map<String, ReplicaCount>>> computeReplicaCounts(List<Row> allRows) {
++    Map<String, Map<String, Map<String, ReplicaCount>>> collVsShardVsTagVsCount = new HashMap<>();
++    for (Row row : allRows) {
+       for (Map.Entry<String, Map<String, List<ReplicaInfo>>> colls : row.collectionVsShardVsReplicas.entrySet()) {
+         String collectionName = colls.getKey();
+         if (!collection.isPass(collectionName)) continue;
 -        collVsShardVsTagVsCount.putIfAbsent(collectionName, new HashMap<>());
 -        Map<String, Map<String, AtomicInteger>> collMap = collVsShardVsTagVsCount.get(collectionName);
++        Map<String, Map<String, ReplicaCount>> collMap = collVsShardVsTagVsCount.computeIfAbsent(collectionName, s -> new HashMap<>());
+         for (Map.Entry<String, List<ReplicaInfo>> shards : colls.getValue().entrySet()) {
+           String shardName = shards.getKey();
+           if (ANY.equals(shard.val)) shardName = ANY;
+           if (!shard.isPass(shardName)) break;
 -          collMap.putIfAbsent(shardName, new HashMap<>());
 -          Map<String, AtomicInteger> tagVsCount = collMap.get(shardName);
++          Map<String, ReplicaCount> tagVsCount = collMap.computeIfAbsent(shardName, s -> new HashMap<>());
+           Object tagVal = row.getVal(tag.name);
 -          tagVsCount.putIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", new AtomicInteger());
++          tagVsCount.computeIfAbsent(tag.isPass(tagVal) ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
+           if (tag.isPass(tagVal)) {
 -            tagVsCount.get(String.valueOf(tagVal)).addAndGet(shards.getValue().size());
++            tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
++          }
+           }
+         }
+       }
+     return collVsShardVsTagVsCount;
+   }
+ 
+   public boolean isStrict() {
+     return strict;
+   }
+ 
+   @Override
+   public String toString() {
+     return Utils.toJSONString(original);
+   }
+ 
+   @Override
+   public void writeMap(EntryWriter ew) throws IOException {
+     for (Map.Entry<String, Object> e : original.entrySet()) ew.put(e.getKey(), e.getValue());
+   }
+ 
+   enum TestStatus {
+     NOT_APPLICABLE, FAIL, PASS
+   }
+ 
 -  private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict"));
++  private static final Set<String> IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict", "type"));
+ 
+   static class ValidateInfo {
+     final Class type;
+     final Set<String> vals;
+     final Number min;
+     final Number max;
+ 
+ 
+     ValidateInfo(Class type, Set<String> vals, Number min, Number max) {
+       this.type = type;
+       this.vals = vals;
+       this.min = min;
 -      if(min != null && !type.isInstance(min)) throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
++      if (min != null && !type.isInstance(min))
++        throw new RuntimeException("wrong min value type, expected: " + type.getName() + " actual: " + min.getClass().getName());
+       this.max = max;
 -      if(max != null && !type.isInstance(max)) throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
++      if (max != null && !type.isInstance(max))
++        throw new RuntimeException("wrong max value type, expected: " + type.getName() + " actual: " + max.getClass().getName());
+     }
+   }
+ 
+ 
+   /**
 -   *
 -   * @param name name of the condition
 -   * @param val value of the condition
++   * @param name      name of the condition
++   * @param val       value of the condition
+    * @param isRuleVal is this provided in the rule
+    * @return actual validated value
+    */
+   public static Object validate(String name, Object val, boolean isRuleVal) {
+     if (val == null) return null;
+     ValidateInfo info = validatetypes.get(name);
+     if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = validatetypes.get("STRING");
+     if (info == null) throw new RuntimeException("Unknown type :" + name);
+     if (info.type == Double.class) {
+       Double num = parseDouble(name, val);
+       if (isRuleVal) {
+         if (info.min != null)
+           if (Double.compare(num, (Double) info.min) == -1)
+             throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+         if (info.max != null)
+           if (Double.compare(num, (Double) info.max) == 1)
+             throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+       }
+       return num;
+     } else if (info.type == Long.class) {
+       Long num = parseLong(name, val);
+       if (isRuleVal) {
+         if (info.min != null)
+           if (num < info.min.longValue())
+             throw new RuntimeException(name + ": " + val + " must be greater than " + info.min);
+         if (info.max != null)
+           if (num > info.max.longValue())
+             throw new RuntimeException(name + ": " + val + " must be less than " + info.max);
+       }
+       return num;
+     } else if (info.type == String.class) {
+       if (isRuleVal && info.vals != null && !info.vals.contains(val))
+         throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(info.vals, ','));
+       return val;
+     } else {
+       throw new RuntimeException("Invalid type ");
+     }
+   }
+ 
+   public static Long parseLong(String name, Object val) {
+     if (val == null) return null;
+     if (val instanceof Long) return (Long) val;
+     Number num = null;
+     if (val instanceof String) {
+       try {
+         num = Long.parseLong(((String) val).trim());
+       } catch (NumberFormatException e) {
+         try {
+           num = Double.parseDouble((String) val);
+         } catch (NumberFormatException e1) {
+           throw new RuntimeException(name + ": " + val + "not a valid number", e);
+         }
+       }
+ 
+     } else if (val instanceof Number) {
+       num = (Number) val;
+     }
+ 
 -    if (num != null)  {
++    if (num != null) {
+       return num.longValue();
+     }
+     throw new RuntimeException(name + ": " + val + "not a valid number");
+   }
+ 
+   public static Double parseDouble(String name, Object val) {
+     if (val == null) return null;
+     if (val instanceof Double) return (Double) val;
+     Number num = null;
+     if (val instanceof String) {
+       try {
+         num = Double.parseDouble((String) val);
+       } catch (NumberFormatException e) {
+         throw new RuntimeException(name + ": " + val + "not a valid number", e);
+       }
+ 
+     } else if (val instanceof Number) {
+       num = (Number) val;
+     }
+ 
 -    if (num != null)  {
++    if (num != null) {
+       return num.doubleValue();
+     }
+     throw new RuntimeException(name + ": " + val + "not a valid number");
+   }
+ 
+   private static final Map<String, ValidateInfo> validatetypes = new HashMap<>();
+ 
+   static {
+     validatetypes.put("collection", new ValidateInfo(String.class, null, null, null));
+     validatetypes.put("shard", new ValidateInfo(String.class, null, null, null));
+     validatetypes.put("replica", new ValidateInfo(Long.class, null, 0L, null));
+     validatetypes.put(ImplicitSnitch.PORT, new ValidateInfo(Long.class, null, 1L, 65535L));
+     validatetypes.put(ImplicitSnitch.DISK, new ValidateInfo(Double.class, null, 0d, Double.MAX_VALUE));
+     validatetypes.put(ImplicitSnitch.NODEROLE, new ValidateInfo(String.class, Collections.singleton("overseer"), null, null));
+     validatetypes.put(ImplicitSnitch.CORES, new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));
+     validatetypes.put(ImplicitSnitch.SYSLOADAVG, new ValidateInfo(Double.class, null, 0d, 100d));
+     validatetypes.put(ImplicitSnitch.HEAPUSAGE, new ValidateInfo(Double.class, null, 0d, null));
+     validatetypes.put("NUMBER", new ValidateInfo(Long.class, null, 0L, Long.MAX_VALUE));//generic number validation
+     validatetypes.put("STRING", new ValidateInfo(String.class, null, null, null));//generic string validation
+     validatetypes.put("node", new ValidateInfo(String.class, null, null, null));
+     for (String ip : ImplicitSnitch.IP_SNITCHES) validatetypes.put(ip, new ValidateInfo(Long.class, null, 0L, 255L));
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
index 0000000,e873625..58972af
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@@ -1,0 -1,52 +1,52 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.io.Closeable;
+ import java.io.IOException;
+ import java.util.Collection;
+ import java.util.List;
+ import java.util.Map;
+ 
+ public interface ClusterDataProvider extends Closeable {
+   /**Get the value of each tag for a given node
+    *
+    * @param node node name
+    * @param tags tag names
+    * @return a map of tag vs value
+    */
+   Map<String, Object> getNodeValues(String node, Collection<String> tags);
+ 
+   /**
+    * Get the details of each replica in a node. It attempts to fetch as much details about
+    * the replica as mentioned in the keys list. It is not necessary to give al details
+    * <p>
+    * the format is {collection:shard :[{replicadetails}]}
+    */
 -  Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
++  Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
+ 
+   Collection<String> getNodes();
+ 
+   /**Get the collection-specific policy
+    */
+   String getPolicyNameByCollection(String coll);
+ 
+   @Override
+   default void close() throws IOException {
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index 0000000,bf9c284..a068253
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@@ -1,0 -1,83 +1,83 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.util.List;
+ 
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
 -import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+ import org.apache.solr.common.util.Pair;
+ 
+ public class MoveReplicaSuggester extends Suggester {
+ 
+   @Override
+   SolrRequest init() {
+     SolrRequest operation = tryEachNode(true);
+     if (operation == null) operation = tryEachNode(false);
+     return operation;
+   }
+ 
+   SolrRequest tryEachNode(boolean strict) {
+     //iterate through elements and identify the least loaded
+     List<Clause.Violation> leastSeriousViolation = null;
+     Integer targetNodeIndex = null;
+     Integer fromNodeIndex = null;
+     ReplicaInfo fromReplicaInfo = null;
+     for (Pair<ReplicaInfo, Row> fromReplica : getValidReplicas(true, true, -1)) {
+       Row fromRow = fromReplica.second();
+       ReplicaInfo replicaInfo = fromReplica.first();
+       String coll = replicaInfo.collection;
+       String shard = replicaInfo.shard;
 -      Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard);
++      Pair<Row, ReplicaInfo> pair = fromRow.removeReplica(coll, shard, replicaInfo.type);
+       Row tmpRow = pair.first();
+       if (tmpRow == null) {
+         //no such replica available
+         continue;
+       }
+       tmpRow.violations.clear();
+ 
+       final int i = getMatrix().indexOf(fromRow);
+       for (int j = getMatrix().size() - 1; j > i; j--) {
+         Row targetRow = getMatrix().get(j);
++        if(!targetRow.isLive) continue;
+         if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) continue;
 -        targetRow = targetRow.addReplica(coll, shard);
++        targetRow = targetRow.addReplica(coll, shard, replicaInfo.type);
+         targetRow.violations.clear();
+         List<Violation> errs = testChangedMatrix(strict, getModifiedMatrix(getModifiedMatrix(getMatrix(), tmpRow, i), targetRow, j));
+         if (!containsNewErrors(errs) && isLessSerious(errs, leastSeriousViolation)) {
+           leastSeriousViolation = errs;
+           targetNodeIndex = j;
+           fromNodeIndex = i;
+           fromReplicaInfo = replicaInfo;
+         }
+       }
+     }
+     if (targetNodeIndex != null && fromNodeIndex != null) {
 -      getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard).first());
 -      getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard));
++      getMatrix().set(fromNodeIndex, getMatrix().get(fromNodeIndex).removeReplica(fromReplicaInfo.collection, fromReplicaInfo.shard, fromReplicaInfo.type).first());
++      getMatrix().set(targetNodeIndex, getMatrix().get(targetNodeIndex).addReplica(fromReplicaInfo.collection, fromReplicaInfo.shard, fromReplicaInfo.type));
+       return new CollectionAdminRequest.MoveReplica(
+           fromReplicaInfo.collection,
+           fromReplicaInfo.name,
+           getMatrix().get(targetNodeIndex).node);
+     }
+     return null;
+   }
+ 
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
index 0000000,e012718..6522cb2
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@@ -1,0 -1,123 +1,124 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.FAIL;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
+ 
+ import java.util.Objects;
+ 
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus;
++import org.apache.solr.common.params.CoreAdminParams;
+ 
+ 
+ public enum Operand {
+   WILDCARD(ANY, Integer.MAX_VALUE) {
+     @Override
+     public TestStatus match(Object ruleVal, Object testVal) {
+       return testVal == null ? NOT_APPLICABLE : PASS;
+     }
+ 
+   },
+   EQUAL("", 0) {
+     @Override
 -    public int _delta(int expected, int actual) {
++    public long _delta(long expected, long actual) {
+       return expected - actual;
+     }
+   },
+   NOT_EQUAL("!", 2) {
+     @Override
+     public TestStatus match(Object ruleVal, Object testVal) {
+       return super.match(ruleVal, testVal) == PASS ? FAIL : PASS;
+     }
+ 
+     @Override
 -    public int _delta(int expected, int actual) {
++    public long _delta(long expected, long actual) {
+       return expected - actual;
+     }
+ 
+   },
+   GREATER_THAN(">", 1) {
+     @Override
+     public TestStatus match(Object ruleVal, Object testVal) {
+       if (testVal == null) return NOT_APPLICABLE;
+       if (ruleVal instanceof Double) {
+         return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? PASS : FAIL;
+       }
+      return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ;
+     }
+ 
+     @Override
 -    protected int _delta(int expected, int actual) {
++    protected long _delta(long expected, long actual) {
+       return actual > expected ? 0 : (expected + 1) - actual;
+     }
+   },
+   LESS_THAN("<", 2) {
+     @Override
+     public TestStatus match(Object ruleVal, Object testVal) {
+       if (testVal == null) return NOT_APPLICABLE;
+       if (ruleVal instanceof Double) {
+         return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? PASS : FAIL;
+       }
+       return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ;
+     }
+ 
+     @Override
 -    protected int _delta(int expected, int actual) {
++    protected long _delta(long expected, long actual) {
+       return actual < expected ? 0 : (expected ) - actual;
+     }
+ 
+   };
+   public final String operand;
+   final int priority;
+ 
+   Operand(String val, int priority) {
+     this.operand = val;
+     this.priority = priority;
+   }
+ 
+   public String toStr(Object expectedVal) {
+     return operand + expectedVal.toString();
+   }
+ 
+   public TestStatus match(Object ruleVal, Object testVal) {
+     return Objects.equals(ruleVal, testVal) ? PASS : FAIL;
+   }
+ 
+   Long getLong(Object o) {
+     if (o instanceof Long) return (Long) o;
+     if(o instanceof Number ) return ((Number) o).longValue();
+     return Long.parseLong(String.valueOf(o));
+ 
+   }
+ 
 -  public Integer delta(Object expected, Object actual) {
++  public Long delta(Object expected, Object actual) {
+     try {
 -      Integer expectedInt = Integer.parseInt(String.valueOf(expected));
 -      Integer actualInt = Integer.parseInt(String.valueOf(actual));
++      Long expectedInt = (Long) Clause.validate(CoreAdminParams.REPLICA, expected, false);
++      Long actualInt = (Long) Clause.validate(CoreAdminParams.REPLICA, actual, false);
+       return _delta(expectedInt, actualInt);
+     } catch (Exception e) {
+       return null;
+     }
+   }
+ 
 -  protected int _delta(int expected, int actual) {
++  protected long _delta(long expected, long actual) {
+     return 0;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 0000000,ccb0dee..451c514
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@@ -1,0 -1,521 +1,499 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.EnumMap;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Objects;
+ import java.util.Set;
+ import java.util.SortedSet;
+ import java.util.TreeSet;
+ import java.util.function.Supplier;
+ import java.util.stream.Collectors;
+ 
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
+ import org.apache.solr.common.IteratorWriter;
+ import org.apache.solr.common.MapWriter;
+ import org.apache.solr.common.params.CollectionParams.CollectionAction;
+ import org.apache.solr.common.util.Pair;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+ 
+ import static java.util.Collections.emptyList;
+ import static java.util.Collections.emptyMap;
+ import static java.util.stream.Collectors.toList;
+ 
+ /*The class that reads, parses and applies policies specified in
+  * autoscaling.json
+  *
+  * Create one instance of this class per unique autoscaling.json.
+  * This is immutable and is thread-safe
+  *
+  * Create a fresh new session for each use
+  *
+  */
+ public class Policy implements MapWriter {
+   public static final String POLICY = "policy";
+   public static final String EACH = "#EACH";
+   public static final String ANY = "#ANY";
+   public static final String CLUSTER_POLICY = "cluster-policy";
+   public static final String CLUSTER_PREFERENCE = "cluster-preferences";
+   public static final Set<String> GLOBAL_ONLY_TAGS = Collections.singleton("cores");
+   final Map<String, List<Clause>> policies = new HashMap<>();
+   final List<Clause> clusterPolicy;
+   final List<Preference> clusterPreferences;
+   final List<String> params;
+ 
+ 
+   public Policy(Map<String, Object> jsonMap) {
+ 
+     clusterPreferences = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_PREFERENCE, emptyList())).stream()
+         .map(Preference::new)
+         .collect(toList());
+     for (int i = 0; i < clusterPreferences.size() - 1; i++) {
+       Preference preference = clusterPreferences.get(i);
+       preference.next = clusterPreferences.get(i + 1);
+     }
+     if (clusterPreferences.isEmpty()) {
+       clusterPreferences.add(new Preference((Map<String, Object>) Utils.fromJSONString("{minimize : cores, precision:1}")));
+     }
+     SortedSet<String> paramsOfInterest = new TreeSet<>();
+     for (Preference preference : clusterPreferences) {
+       if (paramsOfInterest.contains(preference.name.name())) {
+         throw new RuntimeException(preference.name + " is repeated");
+       }
+       paramsOfInterest.add(preference.name.toString());
+     }
+     this.params = new ArrayList<>(paramsOfInterest);
+ 
+     clusterPolicy = ((List<Map<String, Object>>) jsonMap.getOrDefault(CLUSTER_POLICY, emptyList())).stream()
+         .map(Clause::new)
+         .filter(clause -> {
+           clause.addTags(params);
+           return true;
+         })
+         .collect(Collectors.toList());
+ 
+     ((Map<String, List<Map<String, Object>>>) jsonMap.getOrDefault("policies", emptyMap())).forEach((s, l1) ->
+         this.policies.put(s, l1.stream()
+             .map(Clause::new)
+             .filter(clause -> {
+               if (!clause.isPerCollectiontag())
+                 throw new RuntimeException(clause.globalTag.name + " is only allowed in 'cluster-policy'");
+               clause.addTags(params);
+               return true;
+             })
+             .sorted()
+             .collect(toList())));
+   }
+ 
+   public List<Clause> getClusterPolicy() {
+     return clusterPolicy;
+   }
+ 
+   public List<Preference> getClusterPreferences() {
+     return clusterPreferences;
+   }
+ 
+   @Override
+   public void writeMap(EntryWriter ew) throws IOException {
+     if (!policies.isEmpty()) {
+       ew.put("policies", (MapWriter) ew1 -> {
+         for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
+           ew1.put(e.getKey(), e.getValue());
+         }
+       });
+     }
+     if (!clusterPreferences.isEmpty()) {
+       ew.put("preferences", (IteratorWriter) iw -> {
+         for (Preference p : clusterPreferences) iw.add(p);
+       });
+     }
+ 
+   }
+ 
+   /*This stores the logical state of the system, given a policy and
+    * a cluster state.
+    *
+    */
+   public class Session implements MapWriter {
+     final List<String> nodes;
+     final ClusterDataProvider dataProvider;
+     final List<Row> matrix;
+     Set<String> collections = new HashSet<>();
+     List<Clause> expandedClauses;
+     List<Violation> violations = new ArrayList<>();
+ 
+     private Session(List<String> nodes, ClusterDataProvider dataProvider,
+                     List<Row> matrix, List<Clause> expandedClauses) {
+       this.nodes = nodes;
+       this.dataProvider = dataProvider;
+       this.matrix = matrix;
+       this.expandedClauses = expandedClauses;
+     }
+ 
+     Session(ClusterDataProvider dataProvider) {
+       this.nodes = new ArrayList<>(dataProvider.getNodes());
+       this.dataProvider = dataProvider;
+       for (String node : nodes) {
+         collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
+       }
+ 
+       expandedClauses = clusterPolicy.stream()
+           .filter(clause -> !clause.isPerCollectiontag())
+           .collect(Collectors.toList());
+ 
+       for (String c : collections) {
+         addClausesForCollection(dataProvider, c);
+       }
+ 
+       Collections.sort(expandedClauses);
+ 
+       matrix = new ArrayList<>(nodes.size());
+       for (String node : nodes) matrix.add(new Row(node, params, dataProvider));
+       applyRules();
+     }
+ 
+     private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
+       String p = dataProvider.getPolicyNameByCollection(c);
+       if (p != null) {
+         List<Clause> perCollPolicy = policies.get(p);
+         if (perCollPolicy == null)
+           throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
+       }
+       expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+     }
+ 
+     Session copy() {
+       return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses);
+     }
+ 
+     List<Row> getMatrixCopy() {
+       return matrix.stream()
+           .map(Row::copy)
+           .collect(Collectors.toList());
+     }
+ 
+     Policy getPolicy() {
+       return Policy.this;
+ 
+     }
+ 
+     /**
+      * Apply the preferences and conditions
+      */
+     private void applyRules() {
+       if (!clusterPreferences.isEmpty()) {
+         //this is to set the approximate value according to the precision
+         ArrayList<Row> tmpMatrix = new ArrayList<>(matrix);
+         for (Preference p : clusterPreferences) {
+           Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false));
+           p.setApproxVal(tmpMatrix);
+         }
+         //approximate values are set now. Let's do recursive sorting
+         Collections.sort(matrix, (Row r1, Row r2) -> {
+           int result = clusterPreferences.get(0).compare(r1, r2, true);
+           if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false);
+           return result;
+         });
+       }
+ 
+       for (Clause clause : expandedClauses) {
+         List<Violation> errs = clause.test(matrix);
+         violations.addAll(errs);
+       }
+     }
+ 
+     public List<Violation> getViolations() {
+       return violations;
+     }
+ 
+     public Suggester getSuggester(CollectionAction action) {
+       Suggester op = ops.get(action).get();
+       if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
+       op._init(this);
+       return op;
+     }
+ 
+     @Override
+     public void writeMap(EntryWriter ew) throws IOException {
+       for (int i = 0; i < matrix.size(); i++) {
+         Row row = matrix.get(i);
+         ew.put(row.node, row);
+       }
+     }
+ 
+     @Override
+     public String toString() {
+       return Utils.toJSONString(toMap(new LinkedHashMap<>()));
+     }
+ 
+     public List<Row> getSorted() {
+       return Collections.unmodifiableList(matrix);
+     }
+   }
+ 
+ 
+   public Session createSession(ClusterDataProvider dataProvider) {
+     return new Session(dataProvider);
+   }
+ 
+   public enum SortParam {
+     freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100);
+ 
+     public final int min,max;
+ 
+     SortParam(int min, int max) {
+       this.min = min;
+       this.max = max;
+     }
+ 
+     static SortParam get(String m) {
+       for (SortParam p : values()) if (p.name().equals(m)) return p;
+       throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values())));
+     }
+   }
+ 
+   enum Sort {
+     maximize(1), minimize(-1);
+     final int sortval;
+ 
+     Sort(int i) {
+       sortval = i;
+     }
+ 
+     static Sort get(Map<String, Object> m) {
+       if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) {
+         throw new RuntimeException("Cannot have both 'maximize' and 'minimize'");
+       }
+       if (m.containsKey(maximize.name())) return maximize;
+       if (m.containsKey(minimize.name())) return minimize;
+       throw new RuntimeException("must have either 'maximize' or 'minimize'");
+     }
+   }
+ 
+ 
 -  public static class ReplicaInfo implements MapWriter {
 -    final String name;
 -    String core, collection, shard;
 -    Map<String, Object> variables;
 -
 -    public ReplicaInfo(String name, String coll, String shard, Map<String, Object> vals) {
 -      this.name = name;
 -      this.variables = vals;
 -      this.collection = coll;
 -      this.shard = shard;
 -    }
 -
 -    @Override
 -    public void writeMap(EntryWriter ew) throws IOException {
 -      ew.put(name, variables);
 -    }
 -
 -    public String getCore() {
 -      return core;
 -    }
 -
 -    public String getCollection() {
 -      return collection;
 -    }
 -
 -    public String getShard() {
 -      return shard;
 -    }
 -  }
 -
 -
+   /* A suggester is capable of suggesting a collection operation
+    * given a particular session. Before it suggests a new operation,
+    * it ensures that ,
+    *  a) load is reduced on the most loaded node
+    *  b) it causes no new violations
+    *
+    */
+   public static abstract class Suggester {
+     protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
+     Policy.Session session;
+     SolrRequest operation;
+     protected List<Violation> originalViolations = new ArrayList<>();
+     private boolean isInitialized = false;
+ 
+     private void _init(Session session) {
+       this.session = session.copy();
+     }
+ 
+     public Suggester hint(Hint hint, Object value) {
+       if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+         ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).add(value);
+       } else {
 -        hints.put(hint, value);
++        hints.put(hint, value == null ? null : String.valueOf(value));
+       }
+       return this;
+     }
+ 
+     abstract SolrRequest init();
+ 
+ 
+     public SolrRequest getOperation() {
+       if (!isInitialized) {
+         String coll = (String) hints.get(Hint.COLL);
+         String shard = (String) hints.get(Hint.SHARD);
+         // if this is not a known collection from the existing clusterstate,
+         // then add it
+         if (session.matrix.stream().noneMatch(row -> row.collectionVsShardVsReplicas.containsKey(coll))) {
+           session.addClausesForCollection(session.dataProvider, coll);
+           Collections.sort(session.expandedClauses);
+         }
+         if (coll != null) {
+           for (Row row : session.matrix) {
+             if (!row.collectionVsShardVsReplicas.containsKey(coll)) row.collectionVsShardVsReplicas.put(coll, new HashMap<>());
+             if (shard != null) {
+               Map<String, List<ReplicaInfo>> shardInfo = row.collectionVsShardVsReplicas.get(coll);
+               if (!shardInfo.containsKey(shard)) shardInfo.put(shard, new ArrayList<>());
+             }
+           }
+         }
++        Set<String> srcNodes = (Set<String>) hints.get(Hint.SRC_NODE);
++        if (srcNodes != null && !srcNodes.isEmpty()) {
++          // the source node is dead so live nodes may not have it
++          for (String srcNode : srcNodes) {
++            if(session.matrix.stream().noneMatch(row -> row.node.equals(srcNode)))
++            session.matrix.add(new Row(srcNode, session.getPolicy().params, session.dataProvider));
++          }
++        }
+         session.applyRules();
+         originalViolations.addAll(session.getViolations());
+         this.operation = init();
+         isInitialized = true;
+       }
+       return operation;
+     }
+ 
+     public Session getSession() {
+       return session;
+     }
+ 
+     List<Row> getMatrix() {
+       return session.matrix;
+ 
+     }
+ 
+     //check if the fresh set of violations is less serious than the last set of violations
+     boolean isLessSerious(List<Violation> fresh, List<Violation> old) {
+       if (old == null || fresh.size() < old.size()) return true;
+       if (fresh.size() == old.size()) {
+         for (int i = 0; i < fresh.size(); i++) {
+           Violation freshViolation = fresh.get(i);
+           Violation oldViolation = null;
+           for (Violation v : old) {
+             if (v.equals(freshViolation)) oldViolation = v;
+           }
+           if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true;
+         }
+       }
+       return false;
+     }
+ 
+     boolean containsNewErrors(List<Violation> violations) {
+       for (Violation v : violations) {
+         int idx = originalViolations.indexOf(v);
+         if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true;
+       }
+       return false;
+     }
+ 
+     List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
 -      List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
++      List<Pair<ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
+ 
+       if (sortDesc) {
+         if (until == -1) until = getMatrix().size();
+         for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+       } else {
+         if (until == -1) until = 0;
+         for (int i = getMatrix().size() - 1; i >= until; i--)
+           addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas);
+       }
+       return allPossibleReplicas;
+     }
+ 
 -    void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
++    void addReplicaToList(Row r, boolean isSource, List<Pair<ReplicaInfo, Row>> replicaList) {
+       if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
 -      for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
++      for (Map.Entry<String, Map<String, List<ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
+         if (!isAllowed(e.getKey(), Hint.COLL)) continue;
 -        for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
 -          if (!isAllowed(e.getKey(), Hint.SHARD)) continue;
++        for (Map.Entry<String, List<ReplicaInfo>> shard : e.getValue().entrySet()) {
++          if (!isAllowed(e.getKey(), Hint.SHARD)) continue;//todo fix
++          if(shard.getValue() == null || shard.getValue().isEmpty()) continue;
+           replicaList.add(new Pair<>(shard.getValue().get(0), r));
+         }
+       }
+     }
+ 
+     protected List<Violation> testChangedMatrix(boolean strict, List<Row> rows) {
+       List<Violation> errors = new ArrayList<>();
+       for (Clause clause : session.expandedClauses) {
+         if (strict || clause.strict) {
+           List<Violation> errs = clause.test(rows);
+           if (!errs.isEmpty()) {
+             errors.addAll(errs);
+           }
+         }
+       }
+       return errors;
+     }
+ 
+     ArrayList<Row> getModifiedMatrix(List<Row> matrix, Row tmpRow, int i) {
+       ArrayList<Row> copy = new ArrayList<>(matrix);
+       copy.set(i, tmpRow);
+       return copy;
+     }
+ 
+     protected boolean isAllowed(Object v, Hint hint) {
+       Object hintVal = hints.get(hint);
+       if (hint == Hint.TARGET_NODE || hint == Hint.SRC_NODE) {
+         Set set = (Set) hintVal;
+         return set == null || set.contains(v);
+       } else {
+         return hintVal == null || Objects.equals(v, hintVal);
+       }
+     }
+ 
+     public enum Hint {
 -      COLL, SHARD, SRC_NODE, TARGET_NODE
++      COLL, SHARD, SRC_NODE, TARGET_NODE, REPLICATYPE
+     }
+ 
+ 
+   }
+ 
+   public static List<Clause> mergePolicies(String coll,
+                                     List<Clause> collPolicy,
+                                     List<Clause> globalPolicy) {
+ 
+     List<Clause> merged = insertColl(coll, collPolicy);
+     List<Clause> global = insertColl(coll, globalPolicy);
+     merged.addAll(global.stream()
+         .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause)))
+         .collect(Collectors.toList()));
+     return merged;
+   }
+ 
+   /**
+    * Insert the collection name into the clauses where collection is not specified
+    */
+   static List<Clause> insertColl(String coll, Collection<Clause> conditions) {
+     return conditions.stream()
+         .filter(Clause::isPerCollectiontag)
+         .map(clause -> {
+           Map<String, Object> copy = new LinkedHashMap<>(clause.original);
+           if (!copy.containsKey("collection")) copy.put("collection", coll);
+           return new Clause(copy);
+         })
+         .filter(it -> (it.collection.isPass(coll)))
+         .collect(Collectors.toList());
+ 
+   }
+ 
+   private static final Map<CollectionAction, Supplier<Suggester>> ops = new HashMap<>();
+ 
+   static {
+     ops.put(CollectionAction.ADDREPLICA, () -> new AddReplicaSuggester());
+     ops.put(CollectionAction.MOVEREPLICA, () -> new MoveReplicaSuggester());
+   }
+ 
+   public Map<String, List<Clause>> getPolicies() {
+     return policies;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 0000000,deea175..fe6c80b
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@@ -1,0 -1,96 +1,96 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ 
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ 
+ import org.apache.solr.client.solrj.SolrRequest;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
+ import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.params.CoreAdminParams;
+ import org.apache.solr.common.util.Utils;
+ 
+ import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+ 
+ public class PolicyHelper {
+   public static Map<String, List<String>> getReplicaLocations(String collName, Map<String, Object> autoScalingJson,
+                                                               ClusterDataProvider cdp,
+                                                               Map<String, String> optionalPolicyMapping,
+                                                               List<String> shardNames,
+                                                               int repFactor,
+                                                               List<String> nodesList) {
+     Map<String, List<String>> positionMapping = new HashMap<>();
+     for (String shardName : shardNames) positionMapping.put(shardName, new ArrayList<>(repFactor));
+     if (optionalPolicyMapping != null) {
+       final ClusterDataProvider delegate = cdp;
+       cdp = new ClusterDataProvider() {
+         @Override
+         public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+           return delegate.getNodeValues(node, tags);
+         }
+ 
+         @Override
 -        public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
++        public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+           return delegate.getReplicaInfo(node, keys);
+         }
+ 
+         @Override
+         public Collection<String> getNodes() {
+           return delegate.getNodes();
+         }
+ 
+         @Override
+         public String getPolicyNameByCollection(String coll) {
+           return optionalPolicyMapping.containsKey(coll) ?
+               optionalPolicyMapping.get(coll) :
+               delegate.getPolicyNameByCollection(coll);
+         }
+       };
+ 
+     }
+ 
+ 
+     Policy policy = new Policy(autoScalingJson);
+     Policy.Session session = policy.createSession(cdp);
+     for (String shardName : shardNames) {
+       for (int i = 0; i < repFactor; i++) {
+         Policy.Suggester suggester = session.getSuggester(ADDREPLICA)
+             .hint(Hint.COLL, collName)
+             .hint(Hint.SHARD, shardName);
+         if (nodesList != null)  {
+           for (String nodeName : nodesList) {
+             suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
+           }
+         }
+         SolrRequest op = suggester.getOperation();
+         if (op == null) {
+           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules "+ Utils.toJSONString(Utils.getDeepCopy(session.expandedClauses, 4, true)));
+         }
+         session = suggester.getSession();
+         positionMapping.get(shardName).add(op.getParams().get(CoreAdminParams.NODE));
+       }
+     }
+ 
+     return positionMapping;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
index 0000000,bb45628..43fec6b
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@@ -1,0 -1,89 +1,92 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.io.IOException;
+ import java.util.List;
+ import java.util.Map;
+ 
+ import org.apache.solr.common.MapWriter;
+ import org.apache.solr.common.util.StrUtils;
+ import org.apache.solr.common.util.Utils;
+ 
+ public class Preference implements MapWriter {
+   final Policy.SortParam name;
+   Integer precision;
+   final Policy.Sort sort;
+   Preference next;
+   public int idx;
+   private final Map original;
+ 
+   Preference(Map<String, Object> m) {
+     this.original = Utils.getDeepCopy(m,3);
+     sort = Policy.Sort.get(m);
+     name = Policy.SortParam.get(m.get(sort.name()).toString());
+     Object p = m.getOrDefault("precision", 0);
+     precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString());
+     if (precision < 0) {
+       throw new RuntimeException("precision must be a positive value ");
+     }
+     if(precision< name.min || precision> name.max){
+       throw new RuntimeException(StrUtils.formatString("invalid precision value {0} must lie between {1} and {1}",
+           precision, name.min, name.max ) );
+     }
+ 
+   }
+ 
+   // there are 2 modes of compare.
+   // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
+   // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
+   int compare(Row r1, Row r2, boolean useApprox) {
++    if (!r1.isLive && !r2.isLive) return 0;
++    if (!r1.isLive) return -1;
++    if (!r2.isLive) return 1;
+     Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val;
+     Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val;
+     int result = 0;
+     if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2);
+     else if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2);
+     else if (!o1.getClass().getName().equals(o2.getClass().getName()))  {
+       throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString());
+     }
+     return result == 0 ? (next == null ? 0 : next.compare(r1, r2, useApprox)) : sort.sortval * result;
+   }
+ 
+   //sets the new value according to precision in val_
+   void setApproxVal(List<Row> tmpMatrix) {
+     Object prevVal = null;
+     for (Row row : tmpMatrix) {
+       prevVal = row.cells[idx].approxVal =
+           (prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
+               row.cells[idx].val :
+               prevVal;
+     }
+   }
+ 
+   @Override
+   public void writeMap(EntryWriter ew) throws IOException {
+     for (Object o : original.entrySet()) {
+       Map.Entry e = (Map.Entry) o;
+       ew.put(String.valueOf(e.getKey()), e.getValue());
+     }
+   }
+ 
+   public Policy.SortParam getName() {
+     return name;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 0000000,76c8c57..90559c1
mode 000000,100644..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@@ -1,0 -1,120 +1,140 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one or more
+  * contributor license agreements.  See the NOTICE file distributed with
+  * this work for additional information regarding copyright ownership.
+  * The ASF licenses this file to You under the Apache License, Version 2.0
+  * (the "License"); you may not use this file except in compliance with
+  * the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.solr.client.solrj.cloud.autoscaling;
+ 
+ import java.io.IOException;
+ import java.util.ArrayList;
++import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Random;
+ 
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy.ReplicaInfo;
+ import org.apache.solr.common.IteratorWriter;
+ import org.apache.solr.common.MapWriter;
++import org.apache.solr.common.cloud.Replica;
+ import org.apache.solr.common.util.Pair;
+ import org.apache.solr.common.util.Utils;
+ 
+ import static org.apache.solr.common.params.CoreAdminParams.NODE;
+ 
+ 
+ public class Row implements MapWriter {
+   public final String node;
+   final Cell[] cells;
+   public Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas;
+   List<Clause> violations = new ArrayList<>();
+   boolean anyValueMissing = false;
++  boolean isLive = true;
+ 
+   public Row(String node, List<String> params, ClusterDataProvider dataProvider) {
+     collectionVsShardVsReplicas = dataProvider.getReplicaInfo(node, params);
+     if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>();
+     this.node = node;
+     cells = new Cell[params.size()];
 -    Map<String, Object> vals = dataProvider.getNodeValues(node, params);
++    isLive = dataProvider.getNodes().contains(node);
++    Map<String, Object> vals = isLive ? dataProvider.getNodeValues(node, params) : Collections.emptyMap();
+     for (int i = 0; i < params.size(); i++) {
+       String s = params.get(i);
+       cells[i] = new Cell(i, s, Clause.validate(s,vals.get(s), false));
+       if (NODE.equals(s)) cells[i].val = node;
+       if (cells[i].val == null) anyValueMissing = true;
+     }
+   }
+ 
 -  public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations) {
++  public Row(String node, Cell[] cells, boolean anyValueMissing, Map<String,
++      Map<String, List<ReplicaInfo>>> collectionVsShardVsReplicas, List<Clause> violations, boolean isLive) {
+     this.node = node;
++    this.isLive = isLive;
+     this.cells = new Cell[cells.length];
+     for (int i = 0; i < this.cells.length; i++) {
+       this.cells[i] = cells[i].copy();
+ 
+     }
+     this.anyValueMissing = anyValueMissing;
+     this.collectionVsShardVsReplicas = collectionVsShardVsReplicas;
+     this.violations = violations;
+   }
+ 
+   @Override
+   public void writeMap(EntryWriter ew) throws IOException {
+     ew.put(node, (IteratorWriter) iw -> {
+       iw.add((MapWriter) e -> e.put("replicas", collectionVsShardVsReplicas));
+       for (Cell cell : cells) iw.add(cell);
+     });
+   }
+ 
+   Row copy() {
 -    return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations));
++    return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(collectionVsShardVsReplicas, 3), new ArrayList<>(violations), isLive);
+   }
+ 
+   Object getVal(String name) {
+     for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+     return null;
+   }
+ 
+   @Override
+   public String toString() {
+     return node;
+   }
+ 
+   // this adds a replica to the replica info
 -  public Row addReplica(String coll, String shard) {
++  public Row addReplica(String coll, String shard, Replica.Type type) {
+     Row row = copy();
+     Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>());
+     List<ReplicaInfo> replicas = c.computeIfAbsent(shard, k -> new ArrayList<>());
 -    replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, new HashMap<>()));
++    replicas.add(new ReplicaInfo("" + new Random().nextInt(1000) + 1000, coll, shard, type, new HashMap<>()));
+     for (Cell cell : row.cells) {
 -      if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() + 1;
++      if (cell.name.equals("cores")) {
++        cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() + 1;
++      }
+     }
+     return row;
+ 
+   }
+ 
 -  public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard) {
++  public Pair<Row, ReplicaInfo> removeReplica(String coll, String shard, Replica.Type type) {
+     Row row = copy();
+     Map<String, List<ReplicaInfo>> c = row.collectionVsShardVsReplicas.get(coll);
+     if (c == null) return null;
 -    List<ReplicaInfo> s = c.get(shard);
 -    if (s == null || s.isEmpty()) return null;
++    List<ReplicaInfo> r = c.get(shard);
++    if (r == null) return null;
++    int idx = -1;
++    for (int i = 0; i < r.size(); i++) {
++      ReplicaInfo info = r.get(i);
++      if (type == null || info.type == type) {
++        idx = i;
++        break;
++      }
++    }
++    if(idx == -1) return null;
++
+     for (Cell cell : row.cells) {
 -      if (cell.name.equals("cores")) cell.val = ((Number) cell.val).longValue() -1;
++      if (cell.name.equals("cores")) {
++        cell.val = cell.val == null ? 0 : ((Number) cell.val).longValue() - 1;
++      }
+     }
 -    return new Pair(row, s.remove(0));
++    return new Pair(row, r.remove(idx));
+ 
+   }
+ 
+   public Cell[] getCells() {
+     return cells;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4239896e/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------


Mime
View raw message