lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sha...@apache.org
Subject [39/50] [abbrv] lucene-solr:jira/solr-11990: SOLR-12536: autoscaling policy support to equally distribute replicas on the basis of arbitrary properties
Date Sat, 28 Jul 2018 04:50:02 GMT
SOLR-12536: autoscaling policy support to equally distribute replicas on the basis of arbitrary properties


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d87ea6b1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d87ea6b1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d87ea6b1

Branch: refs/heads/jira/solr-11990
Commit: d87ea6b1ccd28e0dd8e30565fe95b2e0a31f82e8
Parents: 6ab3ff8
Author: Noble Paul <noble@apache.org>
Authored: Thu Jul 26 21:32:45 2018 +1000
Committer: Noble Paul <noble@apache.org>
Committed: Thu Jul 26 21:32:45 2018 +1000

----------------------------------------------------------------------
 solr/CHANGES.txt                                |   2 +
 .../client/solrj/cloud/autoscaling/Clause.java  | 115 ++++++++--
 .../client/solrj/cloud/autoscaling/Operand.java |   8 +
 .../client/solrj/cloud/autoscaling/Policy.java  |   2 +-
 .../client/solrj/cloud/autoscaling/Row.java     |   2 +-
 .../solrj/cloud/autoscaling/Suggestion.java     |  93 +++++---
 .../solrj/impl/SolrClientNodeStateProvider.java |  28 +--
 .../solrj/cloud/autoscaling/TestPolicy.java     |  91 ++++++--
 .../solrj/cloud/autoscaling/TestPolicy2.java    | 223 +++++++++++++++++++
 9 files changed, 474 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b35ac22..6c43f14 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -121,6 +121,8 @@ New Features
 * SOLR-12581: the JSON Facet 'relatedness()' aggregate function now supports a 'min_popularity' option
   using the extended type:func syntax (hossman)
 
+* SOLR-12536: autoscaling policy support to equally distribute replicas on the basis of arbitrary properties (noble)
+
 Bug Fixes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index 7bce283..87fecda 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -29,6 +29,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType;
 import org.apache.solr.common.MapWriter;
@@ -38,6 +39,7 @@ import org.apache.solr.common.util.Utils;
 
 import static java.util.Collections.singletonMap;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
@@ -46,6 +48,8 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
 import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
 import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
 import static org.apache.solr.common.params.CoreAdminParams.SHARD;
+import static org.apache.solr.common.util.StrUtils.formatString;
+import static org.apache.solr.common.util.Utils.toJSONString;
 
 /**
  * Represents a set of conditions in the policy
@@ -80,7 +84,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
     if (globalTagName.isPresent()) {
       globalTag = parse(globalTagName.get(), m);
       if (m.size() > 2) {
-        throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + Utils.toJSONString(m));
+        throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + toJSONString(m));
       }
       tag = parse(m.keySet().stream()
           .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s)))
@@ -89,18 +93,18 @@ public class Clause implements MapWriter, Comparable<Clause> {
       collection = parse(COLLECTION, m);
       shard = parse(SHARD, m);
       if (m.get(REPLICA) == null) {
-        throw new RuntimeException(StrUtils.formatString("'replica' is required in {0}", Utils.toJSONString(m)));
+        throw new IllegalArgumentException(formatString("'replica' is required in {0}", toJSONString(m)));
       }
       this.replica = parse(REPLICA, m);
-      if (replica.op == WILDCARD) throw new RuntimeException("replica val cannot be null" + Utils.toJSONString(m));
+      if (replica.op == WILDCARD) throw new IllegalArgumentException("replica val cannot be null" + toJSONString(m));
       m.forEach(this::parseCondition);
     }
     if (tag == null)
-      throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + Utils.toJSONString(m));
+      throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + toJSONString(m));
     if (tag.name.startsWith(Clause.METRICS_PREFIX)) {
       List<String> ss = StrUtils.splitSmart(tag.name, ':');
       if (ss.size() < 3 || ss.size() > 4) {
-        throw new RuntimeException("Invalid metrics: param in " + Utils.toJSONString(m) + " must have at 2 or 3 segments after 'metrics:' separated by ':'");
+        throw new RuntimeException("Invalid metrics: param in " + toJSONString(m) + " must have at 2 or 3 segments after 'metrics:' separated by ':'");
       }
     }
     doPostValidate(collection, shard, replica, tag, globalTag);
@@ -112,7 +116,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
       if (condition == null) continue;
       String err = condition.varType.postValidate(condition);
       if (err != null) {
-        throw new IllegalArgumentException(StrUtils.formatString("Error in clause : {0}, caused by : {1}", Utils.toJSONString(original), err));
+        throw new IllegalArgumentException(formatString("Error in clause : {0}, caused by : {1}", toJSONString(original), err));
       }
     }
   }
@@ -241,8 +245,8 @@ public class Clause implements MapWriter, Comparable<Clause> {
     ComputedType computedType = null;
     Object val = m.get(s);
     ConditionType varType = Suggestion.getTagType(s);
-    if (varType.isHidden) {
-      throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed in a policy rule :  ''{1}''  ", varType.tagName, Utils.toJSONString(m)));
+    if (varType.meta.isHidden()) {
+      throw new IllegalArgumentException(formatString("''{0}'' is not allowed in a policy rule :  ''{1}''  ", varType.tagName, toJSONString(m)));
     }
     try {
       String conditionName = s.trim();
@@ -250,14 +254,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
       if (val == null) {
         operand = WILDCARD;
         expectedVal = Policy.ANY;
+      } else if (val instanceof List) {
+        if (!varType.meta.supportArrayVals()) {
+          throw new IllegalArgumentException(formatString("array values are not supported for {0} in clause {1}",
+              conditionName, toJSONString(m)));
+        }
+        expectedVal = readListVal(m, (List) val, varType, conditionName);
+        operand = Operand.IN;
       } else if (val instanceof String) {
         String strVal = ((String) val).trim();
         val = strVal;
-        if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
-        else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
-        else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
-        else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
-        else operand = Operand.EQUAL;
+        operand = getOperand(strVal);
         strVal = strVal.substring(Operand.EQUAL == operand || WILDCARD == operand ? 0 : 1);
         for (ComputedType t : ComputedType.values()) {
           String changedVal = t.match(strVal);
@@ -265,15 +272,14 @@ public class Clause implements MapWriter, Comparable<Clause> {
             computedType = t;
             strVal = changedVal;
             if (varType == null || !varType.supportedComputedTypes.contains(computedType)) {
-              throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not allowed for variable :  ''{1}'' , in clause : ''{2}'' ",
-                  t, conditionName, Utils.toJSONString(m)));
+              throw new IllegalArgumentException(formatString("''{0}'' is not allowed for variable :  ''{1}'' , in clause : ''{2}'' ",
+                  t, conditionName, toJSONString(m)));
             }
           }
         }
         if (computedType == null && ((String) val).charAt(0) == '#' && !varType.wildCards.contains(val)) {
-          throw new IllegalArgumentException(StrUtils.formatString("''{0}'' is not an allowed value for ''{1}'' , in clause : ''{2}'' . Supported value is : {3}",
-              val, conditionName, Utils.toJSONString(m), varType.wildCards));
-
+          throw new IllegalArgumentException(formatString("''{0}'' is not an allowed value for ''{1}'' , in clause : ''{2}'' . Supported value is : {3}",
+              val, conditionName, toJSONString(m), varType.wildCards));
 
         }
         operand = varType == null ? operand : varType.getOperand(operand, strVal, computedType);
@@ -293,6 +299,49 @@ public class Clause implements MapWriter, Comparable<Clause> {
     }
   }
 
+  private List readListVal(Map m, List val, ConditionType varType, String conditionName) {
+    List list = val;
+    list = (List) list.stream()
+        .map(it -> varType.validate(conditionName, it, true))
+        .map(it -> {
+          if (it instanceof String) {
+            String trim = ((String) it).trim();
+            if (trim.isEmpty())
+              throw new IllegalArgumentException(formatString("{0} cannot have an empty string value in clause : {1}",
+                  conditionName, toJSONString(m)));
+            return trim;
+          } else return it;
+        }).filter(it -> it == null ? false : true)
+        .collect(Collectors.toList());
+    if (list.isEmpty())
+      throw new IllegalArgumentException(formatString("{0} cannot have an empty list value in clause : {1}",
+          conditionName, toJSONString(m)));
+    for (Object o : list) {
+      if (o instanceof String) {
+        if (getOperand((String) o) != EQUAL) {
+          throw new IllegalArgumentException(formatString("No operators are supported in collection values in condition : {0} in clause : {1}",
+              conditionName, toJSONString(m)));
+        }
+      }
+    }
+    if (list.size() < 2) {
+      throw new IllegalArgumentException(formatString("Array should have more than one value in  condition : {0} in clause : {1}",
+          conditionName, toJSONString(m)));
+
+    }
+    return list;
+  }
+
+  private Operand getOperand(String strVal) {
+    Operand operand;
+    if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD;
+    else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL;
+    else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN;
+    else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN;
+    else operand = Operand.EQUAL;
+    return operand;
+  }
+
   public List<Violation> test(Policy.Session session) {
     ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
     Suggestion.ViolationCtx ctx = new Suggestion.ViolationCtx(this, session.matrix, computedValueEvaluator);
@@ -305,17 +354,17 @@ public class Clause implements MapWriter, Comparable<Clause> {
           computedValueEvaluator.shardName = shardVsCount.getKey();
           if (!shard.isPass(computedValueEvaluator.shardName)) continue;
           for (Map.Entry<String, ReplicaCount> counts : shardVsCount.getValue().entrySet()) {
-            if(tag.varType.isPerNodeValue) computedValueEvaluator.node = counts.getKey();
+            if (tag.varType.meta.isNodeSpecificVal()) computedValueEvaluator.node = counts.getKey();
             SealedClause sealedClause = getSealedClause(computedValueEvaluator);
             ReplicaCount replicas = counts.getValue();
             if (!sealedClause.replica.isPass(replicas)) {
               Violation violation = new Violation(sealedClause,
                   computedValueEvaluator.collName,
                   computedValueEvaluator.shardName,
-                  tag.varType.isPerNodeValue ? computedValueEvaluator.node : null,
+                  tag.varType.meta.isNodeSpecificVal() ? computedValueEvaluator.node : null,
                   counts.getValue(),
                   sealedClause.getReplica().delta(replicas),
-                  tag.varType.isPerNodeValue? null:  counts.getKey());
+                  tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
               tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
             }
           }
@@ -353,7 +402,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
           computedValueEvaluator.shardName = shardName;
           SealedClause sealedClause = getSealedClause(computedValueEvaluator);
           Condition t = sealedClause.getTag();
-          if(t.varType.isPerNodeValue){
+          if (t.varType.meta.isNodeSpecificVal()) {
             boolean pass = t.getOperand().match(t.val, tagVal) == TestStatus.PASS;
             tagVsCount.computeIfAbsent(row.node, s -> new ReplicaCount());
             if(pass) {
@@ -372,6 +421,23 @@ public class Clause implements MapWriter, Comparable<Clause> {
     return collVsShardVsTagVsCount;
   }
 
+  public boolean isMatch(ReplicaInfo r, String collection, String shard) {
+    if (type != null && r.getType() != type) return false;
+    if (r.getCollection().equals(collection)) {
+      if (this.shard == null || this.shard.val.equals(Policy.ANY)) return true;
+      else if (this.shard.val.equals(Policy.EACH) && r.getShard().equals(shard)) return true;
+      else return this.shard.val.equals(r.getShard()) && r.getShard().equals(shard);
+    }
+    return false;
+  }
+
+  boolean matchShard(String replicaShard, String shardInContext) {
+    if (shard == null || shard.val.equals(ANY)) return true;
+    if (shard.val.equals(Policy.EACH) && replicaShard.equals(shardInContext)) return true;
+    if (shard.val.equals(replicaShard)) return true;
+    return false;
+  }
+
   enum ComputedType {
     NULL(),
     EQUAL() {
@@ -555,7 +621,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
 
   @Override
   public String toString() {
-    return Utils.toJSONString(original);
+    return toJSONString(original);
   }
 
   @Override
@@ -625,6 +691,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
 
   public static Double parseDouble(String name, Object val) {
     if (val == null) return null;
+    if (val instanceof RangeVal) val = ((RangeVal) val).actual;
     if (val instanceof Double) return (Double) val;
     Number num = null;
     if (val instanceof String) {
@@ -661,7 +728,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
     }
 
     public Double delta(double v) {
-      if (actual != null) return v - actual.doubleValue();
+//      if (actual != null) return v - actual.doubleValue();
       if (v >= max.doubleValue()) return v - max.doubleValue();
       if (v <= min.doubleValue()) return v - min.doubleValue();
       return 0d;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
index abd6531..b458646 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java
@@ -17,6 +17,7 @@
 
 package org.apache.solr.client.solrj.cloud.autoscaling;
 
+import java.util.List;
 import java.util.Objects;
 
 import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus;
@@ -76,6 +77,13 @@ public enum Operand {
       return actual - expected;
     }
   },
+  IN("", 0) {
+    @Override
+    public TestStatus match(Object ruleVal, Object testVal) {
+      List l = (List) ruleVal;
+      return (l.contains(testVal)) ?  PASS: FAIL;
+    }
+  },
   RANGE_NOT_EQUAL("", 2) {
     @Override
     public TestStatus match(Object ruleVal, Object testVal) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 9addc67..a5f57bb 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -539,7 +539,7 @@ public class Policy implements MapWriter {
           .collect(Collectors.toList());
     }
 
-    Policy getPolicy() {
+    public Policy getPolicy() {
       return Policy.this;
 
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
index 1adfdbf..c00a90eb 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java
@@ -106,7 +106,7 @@ public class Row implements MapWriter {
 
   @Override
   public String toString() {
-    return node;
+    return jsonStr();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index c61f371..a481a40 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -23,6 +23,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -77,6 +78,8 @@ public class Suggestion {
 
     double max() default -1d;
 
+    boolean supportArrayVals() default false;
+
     String metricsKey() default NULL;
 
     ComputedType[] computedValues() default ComputedType.NULL;
@@ -207,13 +210,20 @@ public class Suggestion {
       public String postValidate(Clause.Condition condition) {
         if (condition.computedType == ComputedType.EQUAL) {
           if (condition.getClause().tag != null &&
-              condition.getClause().tag.varType == NODE &&
-              condition.getClause().tag.op == Operand.WILDCARD) {
+//              condition.getClause().tag.varType == NODE &&
+              (condition.getClause().tag.op == Operand.WILDCARD || condition.getClause().tag.op == Operand.IN)) {
             return null;
           } else {
             return "'replica': '#EQUAL` must be used with 'node':'#ANY'";
           }
         }
+        if (condition.computedType == ComputedType.ALL) {
+          if (condition.getClause().tag != null && (condition.getClause().getTag().op == Operand.IN ||
+              condition.getClause().getTag().op == Operand.WILDCARD)) {
+            return StrUtils.formatString("array value or wild card cannot be used for tag {0} with replica : '#ALL'",
+                condition.getClause().tag.getName());
+          }
+        }
         return null;
       }
 
@@ -223,8 +233,9 @@ public class Suggestion {
           return Double.valueOf(getRelevantReplicasCount(session, cv, collection, shard));
         if (cv.computedType == ComputedType.EQUAL) {
           int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard);
-          if (relevantReplicasCount == 0) return 0;
-          return (double) session.matrix.size() / (double) relevantReplicasCount;
+          double bucketsCount = getNumBuckets(session, cv.getClause());
+          if (relevantReplicasCount == 0 || bucketsCount == 0) return 0;
+          return (double) relevantReplicasCount / bucketsCount;
         } else if (cv.computedType == ComputedType.PERCENT) {
           return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv);
         } else {
@@ -232,33 +243,60 @@ public class Suggestion {
 
         }
       }
+
+      private int getNumBuckets(Policy.Session session, Clause clause) {
+        if (clause.getTag().getOperand() == Operand.IN) {
+          return ((Collection) clause.getTag().val).size();
+        } else if (clause.getTag().getOperand() == Operand.WILDCARD) {
+          if (clause.getTag().varType == NODE) return session.matrix.size();
+          Set uniqueVals = new HashSet();
+          for (Row matrix : session.matrix) {
+            Object val = matrix.getVal(clause.getTag().name);
+            if (val != null) uniqueVals.add(val);
+          }
+          return uniqueVals.size();
+        } else {
+          throw new IllegalArgumentException("Invalid operand for the tag in  " + clause);
+        }
+
+      }
     },
     @Meta(name = ImplicitSnitch.PORT,
         type = Long.class,
         min = 1,
-        max = 65535)
+        max = 65535,
+        supportArrayVals = true,
+        wildCards = Policy.EACH
+    )
     PORT(),
     @Meta(name = "ip_1",
         type = Long.class,
         min = 0,
-        max = 255)
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
     IP_1(),
     @Meta(name = "ip_2",
         type = Long.class,
         min = 0,
-        max = 255)
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
     IP_2(),
     @Meta(name = "ip_3",
         type = Long.class,
         min = 0,
-        max = 255)
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
     IP_3(),
     @Meta(name = "ip_4",
         type = Long.class,
         min = 0,
-        max = 255)
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
     IP_4(),
-
     @Meta(name = ImplicitSnitch.DISK,
         type = Double.class,
         min = 0,
@@ -452,15 +490,18 @@ public class Suggestion {
         type = Long.class,
         min = 0)
     NUMBER(),
+
     @Meta(name = "STRING",
         type = String.class,
-        wildCards = Policy.EACH)
+        wildCards = Policy.EACH,
+        supportArrayVals = true)
     STRING(),
 
     @Meta(name = "node",
         type = String.class,
         isNodeSpecificVal = true,
-        wildCards = {Policy.ANY, Policy.EACH})
+        wildCards = {Policy.ANY, Policy.EACH},
+        supportArrayVals = true)
     NODE() {
       @Override
       public void getSuggestions(SuggestionCtx ctx) {
@@ -495,7 +536,8 @@ public class Suggestion {
 
     @Meta(name = ImplicitSnitch.DISKTYPE,
         type = String.class,
-        enumVals = {"ssd", "rotational"})
+        enumVals = {"ssd", "rotational"},
+        supportArrayVals = true)
     DISKTYPE() {
       @Override
       public void getSuggestions(SuggestionCtx ctx) {
@@ -511,12 +553,10 @@ public class Suggestion {
     public final Number min;
     public final Number max;
     public final Boolean additive;
-    public final boolean isHidden;
     public final Set<String> wildCards;
     public final String perReplicaValue;
     public final Set<String> associatedPerNodeValues;
     public final String metricsAttribute;
-    public final boolean isPerNodeValue;
     public final Set<ComputedType> supportedComputedTypes;
 
 
@@ -539,14 +579,16 @@ public class Suggestion {
       this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
       this.additive = meta.isAdditive();
       this.metricsAttribute = readStr(meta.metricsKey());
-      this.isPerNodeValue = meta.isNodeSpecificVal();
       this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ?
           emptySet() :
           unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
-      this.isHidden = meta.isHidden();
       this.wildCards = readSet(meta.wildCards());
     }
 
+    public String getTagName() {
+      return meta.name();
+    }
+
     private String readStr(String s) {
       return NULL.equals(s) ? null : s;
     }
@@ -567,7 +609,7 @@ public class Suggestion {
 
     public void addViolatingReplicas(ViolationCtx ctx) {
       for (Row row : ctx.allRows) {
-        if (ctx.clause.tag.varType.isPerNodeValue && !row.node.equals(ctx.tagKey)) continue;
+        if (ctx.clause.tag.varType.meta.isNodeSpecificVal() && !row.node.equals(ctx.tagKey)) continue;
         collectViolatingReplicas(ctx, row);
       }
     }
@@ -645,7 +687,7 @@ public class Suggestion {
   }
 
   private static void collectViolatingReplicas(ViolationCtx ctx, Row row) {
-    if (ctx.clause.tag.varType.isPerNodeValue) {
+    if (ctx.clause.tag.varType.meta.isNodeSpecificVal()) {
       row.forEachReplica(replica -> {
         if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard())) {
           ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
@@ -656,7 +698,7 @@ public class Suggestion {
       row.forEachReplica(replica -> {
         if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
         if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
-        if (!ctx.currentViolation.matchShard(replica.getShard())) return;
+        if(!ctx.currentViolation.getClause().matchShard(replica.getShard(), ctx.currentViolation.shard)) return;
         if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
           return;
         ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
@@ -672,12 +714,8 @@ public class Suggestion {
     Clause clause = cv.getClause();
     for (Row row : session.matrix) {
       row.forEachReplica(replicaInfo -> {
-        if (replicaInfo.getCollection().equals(collection)) {
-          if (clause.getShard() ==null || clause.getShard().op == Operand.WILDCARD || replicaInfo.getShard().equals(shard)) {
-            if (cv.getClause().type == null || replicaInfo.getType() == cv.getClause().type)
-              totalReplicasOfInterest.incrementAndGet();
-          }
-        }
+        if (clause.isMatch(replicaInfo, collection, shard))
+          totalReplicasOfInterest.incrementAndGet();
       });
     }
     return totalReplicasOfInterest.get();
@@ -719,9 +757,6 @@ public class Suggestion {
     }
   }
 
-  /*public static final Map<String, String> tagVsPerReplicaVal = Stream.of(ConditionType.values())
-      .filter(tag -> tag.perReplicaValue != null)
-      .collect(Collectors.toMap(tag -> tag.tagName, tag -> tag.perReplicaValue));*/
   static {
     for (Suggestion.ConditionType t : Suggestion.ConditionType.values()) Suggestion.validatetypes.put(t.tagName, t);
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 92183b8..a770102 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -72,7 +72,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
 
 
   private final CloudSolrClient solrClient;
-  private final Map<String, Map<String, Map<String, List<ReplicaInfo>>>> nodeVsCollectionVsShardVsReplicaInfo = new HashMap<>();
+  protected final Map<String, Map<String, Map<String, List<ReplicaInfo>>>> nodeVsCollectionVsShardVsReplicaInfo = new HashMap<>();
   private Map<String, Object> snitchSession = new HashMap<>();
   private Map<String, Map> nodeVsTags = new HashMap<>();
 
@@ -140,28 +140,26 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
   public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
     Map<String, Map<String, List<ReplicaInfo>>> result = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> emptyMap());
     if (!keys.isEmpty()) {
-      Map<String, Pair<String, ReplicaInfo>> keyVsReplica = new HashMap<>();
+      Map<String, Pair<String, ReplicaInfo>> metricsKeyVsTagReplica = new HashMap<>();
       Row.forEachReplica(result, r -> {
         for (String key : keys) {
-          if (r.getVariables().containsKey(key)) continue;
-          String perReplicaAttrKeyPrefix = "solr.core." + r.getCollection() + "." + r.getShard() + "." + Utils.parseMetricsReplicaName(r.getCollection(), r.getCore()) + ":";
+          if (r.getVariables().containsKey(key)) continue;// it's already collected
+          String perReplicaMetricsKey = "solr.core." + r.getCollection() + "." + r.getShard() + "." + Utils.parseMetricsReplicaName(r.getCollection(), r.getCore()) + ":";
           Suggestion.ConditionType tagType = Suggestion.getTagType(key);
           String perReplicaValue = key;
           if (tagType != null) {
             perReplicaValue = tagType.metricsAttribute;
             perReplicaValue = perReplicaValue == null ? key : perReplicaValue;
           }
-          perReplicaAttrKeyPrefix += perReplicaValue;
-          keyVsReplica.put(perReplicaAttrKeyPrefix, new Pair<>(key, r));
+          perReplicaMetricsKey += perReplicaValue;
+          metricsKeyVsTagReplica.put(perReplicaMetricsKey, new Pair<>(key, r));
         }
       });
 
-      if (!keyVsReplica.isEmpty()) {
-        Map<String, Object> tags = fetchReplicaMetrics(node,
-            keyVsReplica.entrySet().stream()
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey)));
-        tags.forEach((k, o) -> {
-          Pair<String, ReplicaInfo> p = keyVsReplica.get(k);
+      if (!metricsKeyVsTagReplica.isEmpty()) {
+        Map<String, Object> tagValues = fetchReplicaMetrics(node, metricsKeyVsTagReplica);
+        tagValues.forEach((k, o) -> {
+          Pair<String, ReplicaInfo> p = metricsKeyVsTagReplica.get(k);
           Suggestion.ConditionType validator = Suggestion.getTagType(p.first());
           if (validator != null) o = validator.convertVal(o);
           if (p.second() != null) p.second().getVariables().put(p.first(), o);
@@ -172,9 +170,11 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
     return result;
   }
 
-  protected  Map<String,Object> fetchReplicaMetrics(String solrNode, Map<String, Object> metricsKeyVsTag) {
+  protected Map<String, Object> fetchReplicaMetrics(String node, Map<String, Pair<String, ReplicaInfo>> metricsKeyVsTagReplica) {
+    Map<String, Object> collect = metricsKeyVsTagReplica.entrySet().stream()
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey));
     ClientSnitchCtx ctx = new ClientSnitchCtx(null, null, emptyMap(), solrClient);
-    fetchReplicaMetrics(solrNode, ctx,metricsKeyVsTag);
+    fetchReplicaMetrics(node, ctx, collect);
     return ctx.getTags();
 
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 0d250fd..feae38b 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -72,6 +72,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.REPLICA;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
@@ -79,7 +80,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
 
 public class TestPolicy extends SolrTestCaseJ4 {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-  private Suggester createSuggester(SolrCloudManager cloudManager, Map jsonObj, Suggester seed) throws IOException, InterruptedException {
+  static Suggester createSuggester(SolrCloudManager cloudManager, Map jsonObj, Suggester seed) throws IOException, InterruptedException {
     Policy.Session session = null;
     if (seed != null) session = seed.session;
     else {
@@ -96,7 +97,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
     return result;
   }
 
-  private SolrCloudManager createCloudManager(Map jsonObj) {
+  static SolrCloudManager createCloudManager(Map jsonObj) {
     return cloudManagerWithData(jsonObj);
   }
 
@@ -282,9 +283,35 @@ public class TestPolicy extends SolrTestCaseJ4 {
         () -> Clause.create("{replica:'<3', shard: '#ANV', node:'#ANY'}"));
     expectThrows(IllegalArgumentException.class,
         () -> Clause.create("{replica:'<3', shard: '#EACH', node:'#E4CH'}"));
+    try {
+      Clause.create("{replica:0, 'ip_1':'<30%'}");
+      fail("Expected exception");
+    } catch (Exception e) {
+      assertTrue(e.getMessage().contains("'%' is not allowed for variable :  'ip_1'"));
+    }
 
     clause = Clause.create("{replica: '#ALL',  freedisk:'>20%'}");
     clause = Clause.create("{replica: '#ALL',  sysprop.zone :'west'}");
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: [3,4] ,  freedisk:'>20'}"));
+    clause = Clause.create("{replica: 3 ,  port:[8983, 7574]}");
+    assertEquals(Operand.IN, clause.tag.op);
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: 3 ,  sysprop.zone :['east', ' ', 'west']}"));
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: 3 ,  sysprop.zone :[]}"));
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: 3 ,  sysprop.zone :['!east','west']}"));
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: '#ALL' , shard: '#EACH' , sysprop.zone:[east, west]}"));
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: '#ALL' , shard: '#EACH' , sysprop.zone:'#EACH'}"));
+    clause = Clause.create("{replica: '#EQUAL' , shard: '#EACH' , sysprop.zone:[east, west]}");
+    assertEquals(Clause.ComputedType.EQUAL, clause.replica.computedType);
+    assertEquals(Operand.IN, clause.tag.op);
+    expectThrows(IllegalArgumentException.class,
+        () -> Clause.create("{replica: '#EQUAL' , shard: '#EACH' , sysprop.zone:[east]}"));
+
   }
 
 
@@ -375,10 +402,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
       }
 
       @Override
-      protected Map<String, Object> fetchReplicaMetrics(String solrNode, Map<String, Object> metricsKeyVsTag) {
+      protected Map<String, Object> fetchReplicaMetrics(String node, Map<String, Pair<String, ReplicaInfo>> metricsKeyVsTagReplica) {
         //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes
         Map<String, Object> result = new HashMap<>();
-        metricsKeyVsTag.forEach((k, v) -> {
+        metricsKeyVsTagReplica.forEach((k, v) -> {
           if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100);
         });
 
@@ -417,9 +444,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
     Violation violation = violations.get(0);
     assertEquals("node1", violation.node);
     RangeVal val = (RangeVal) violation.getClause().replica.val;
-    assertEquals(0.0, val.min);
-    assertEquals(1.0, val.max);
-    assertEquals(0, Preference.compareWithTolerance(val.actual.doubleValue(), 0.833, 1));
+    assertEquals(1.0d, val.min.doubleValue(), 0.01);
+    assertEquals(2.0, val.max.doubleValue(), 0.01);
+    assertEquals(1.2d, val.actual.doubleValue(), 0.01d);
+    assertEquals(1, violation.replicaCountDelta.doubleValue(), 0.01);
     assertEquals(3, violation.getViolatingReplicas().size());
     Set<String> expected = ImmutableSet.of("r1", "r3", "r5");
     for (Violation.ReplicaInfoAndErr replicaInfoAndErr : violation.getViolatingReplicas()) {
@@ -476,13 +504,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
     assertFalse(c.tag.isPass("12.6"));
     assertFalse(c.tag.isPass(12.6d));
 
-    try {
-      c = Clause.create("{replica:0, 'ip_1':'<30%'}");
-      fail("Expected exception");
-    } catch (Exception e) {
-      assertTrue(e.getMessage().contains("'%' is not allowed for variable :  'ip_1'"));
-
-    }
+    c = Clause.create("{replica: '<3', sysprop.zone : [east, west]}");
+    assertTrue(c.tag.isPass("east"));
+    assertTrue(c.tag.isPass("west"));
+    assertFalse(c.tag.isPass("south"));
 
   }
 
@@ -667,7 +692,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
     return cloudManagerWithData((Map) Utils.fromJSONString(data));
   }
 
-  private static SolrCloudManager cloudManagerWithData(Map m) {
+  static SolrCloudManager cloudManagerWithData(Map m) {
     Map replicaInfo = (Map) m.get("replicaInfo");
     replicaInfo.forEach((node, val) -> {
       Map m1 = (Map) val;
@@ -1743,17 +1768,16 @@ public class TestPolicy extends SolrTestCaseJ4 {
         "  'replicaInfo':{" +
         "    '10.0.0.6:7574_solr':{}," +
         "    '10.0.0.6:8983_solr':{'mycoll1':{" +
-        "        'shard2':[{'core_node2':{'type':'NRT'}}]," +
-        "        'shard1':[{'core_node1':{'type':'NRT'}}]}}}," +
+        "        'shard1':[{'core_node1':{'type':'NRT'}},{'core_node2':{'type':'NRT'}},{'core_node3':{'type':'NRT'}}]}}}," +
         "  'nodeValues':{" +
         "    '10.0.0.6:7574_solr':{" +
         "      'node':'10.0.0.6:7574_solr'," +
         "      'cores':0}," +
         "    '10.0.0.6:8983_solr':{" +
         "      'node':'10.0.0.6:8983_solr'," +
-        "      'cores':2}}}";
+        "      'cores':3}}}";
     String autoScalingjson = "  { cluster-policy:[" +
-        "    { replica :'<51%',  node:'#ANY'}]," +
+        "    { replica :'51%', shard:'#EACH', node:'#ANY'}]," +
         "  cluster-preferences :[{ minimize : cores }]}";
 
 
@@ -1761,8 +1785,32 @@ public class TestPolicy extends SolrTestCaseJ4 {
     Policy.Session session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(dataproviderdata));
     List<Violation> violations = session.getViolations();
     assertEquals(1, violations.size());
+    assertEquals(1.0d, violations.get(0).replicaCountDelta, 0.01);
+    assertEquals(1.53d, ((RangeVal) violations.get(0).getClause().getReplica().val).actual);
+
+
+    dataproviderdata = "{" +
+        "  'liveNodes':[" +
+        "    '10.0.0.6:7574_solr'," +
+        "    '10.0.0.6:8983_solr']," +
+        "  'replicaInfo':{" +
+        "    '10.0.0.6:7574_solr':{}," +
+        "    '10.0.0.6:8983_solr':{'mycoll1':{" +
+        "        'shard2':[{'core_node2':{'type':'NRT'}}]," +
+        "        'shard1':[{'core_node1':{'type':'NRT'}}]}}}," +
+        "  'nodeValues':{" +
+        "    '10.0.0.6:7574_solr':{" +
+        "      'node':'10.0.0.6:7574_solr'," +
+        "      'cores':0}," +
+        "    '10.0.0.6:8983_solr':{" +
+        "      'node':'10.0.0.6:8983_solr'," +
+        "      'cores':2}}}";
+
+    session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(dataproviderdata));
+    violations = session.getViolations();
+    assertEquals(0, violations.size());
     autoScalingjson = "  { cluster-policy:[" +
-        "    { replica :'<51%', shard: '#EACH' , node:'#ANY'}]," +
+        "    { replica :'51%', shard: '#EACH' , node:'#ANY'}]," +
         "  cluster-preferences :[{ minimize : cores }]}";
     autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
     session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(dataproviderdata));
@@ -1909,6 +1957,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
         "  cluster-preferences :[{ minimize : cores, precision : 2 }]}";
     cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
     violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
+    assertEquals(1, violations.size());
     assertEquals(-4, violations.get(0).replicaCountDelta, 0.1);
     assertEquals(1, violations.size());
     assertEquals(4, violations.get(0).getViolatingReplicas().size());
@@ -2807,7 +2856,7 @@ public void testUtilizeNodeFailure2() throws Exception {
 
     List l = (List) ((Map) Utils.fromJSONString(rowsData)).get("sortedNodes");
     List<Suggestion.ConditionType> params = new ArrayList<>();
-    params.add(Suggestion.ConditionType.CORES);
+    params.add(CORES);
     params.add(Suggestion.ConditionType.FREEDISK);
     params.add(Suggestion.ConditionType.SYSLOADAVG);
     params.add(Suggestion.ConditionType.NODE);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d87ea6b1/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
new file mode 100644
index 0000000..9c5528a
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.cloud.NodeStateProvider;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Collections.emptyMap;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
+
+public class TestPolicy2 extends SolrTestCaseJ4 {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  public void testEqualOnNonNode() {
+    String state = "{" +
+        "  'coll1': {" +
+        "    'router': {" +
+        "      'name': 'compositeId'" +
+        "    }," +
+        "    'shards': {" +
+        "      'shard1': {" +
+        "        'range': '80000000-ffffffff'," +
+        "        'replicas': {" +
+        "          'r1': {" +
+        "            'core': 'r1'," +
+        "            'base_url': 'http://10.0.0.4:8983/solr'," +
+        "            'node_name': 'node1'," +
+        "            'state': 'active'" +
+        "          }," +
+        "          'r2': {" +
+        "            'core': 'r2'," +
+        "            'base_url': 'http://10.0.0.4:7574/solr'," +
+        "            'node_name': 'node2'," +
+        "            'state': 'active'" +
+        "          }" +
+        "        }" +
+        "      }," +
+        "      'shard2': {" +
+        "        'range': '0-7fffffff'," +
+        "        'replicas': {" +
+        "          'r3': {" +
+        "            'core': 'r3'," +
+        "            'base_url': 'http://10.0.0.4:8983/solr'," +
+        "            'node_name': 'node1'," +
+        "            'state': 'active'" +
+        "          }," +
+        "          'r4': {" +
+        "            'core': 'r4'," +
+        "            'base_url': 'http://10.0.0.4:8987/solr'," +
+        "            'node_name': 'node4'," +
+        "            'state': 'active'" +
+        "          }," +
+        "          'r6': {" +
+        "            'core': 'r6'," +
+        "            'base_url': 'http://10.0.0.4:8989/solr'," +
+        "            'node_name': 'node3'," +
+        "            'state': 'active'" +
+        "          }," +
+        "          'r5': {" +
+        "            'core': 'r5'," +
+        "            'base_url': 'http://10.0.0.4:8983/solr'," +
+        "            'node_name': 'node1'," +
+        "            'state': 'active'" +
+        "          }" +
+        "        }" +
+        "      }" +
+        "    }" +
+        "  }" +
+        "}";
+    String metaData =
+        "  {'nodeValues':{" +
+            "    'node1':{'cores' : 3, 'freedisk' : 700, 'totaldisk' :1000, 'sysprop.zone' : 'east'}," +
+            "    'node2':{'cores' : 1, 'freedisk' : 900, 'totaldisk' :1000, 'sysprop.zone' : 'west'}," +
+            "    'node3':{'cores' : 1, 'freedisk' : 900, 'totaldisk' :1000, 'sysprop.zone': 'east'}," +
+            "    'node4':{'cores' : 1, 'freedisk' : 900, 'totaldisk' :1000, 'sysprop.zone': 'west'}" +
+            "  }," +
+            "  'replicaValues':[" +
+            "    {'INDEX.sizeInGB': 100, core : r1}," +
+            "    {'INDEX.sizeInGB': 100, core : r2}," +
+            "    {'INDEX.sizeInGB': 100, core : r3}," +
+            "    {'INDEX.sizeInGB': 100, core : r4}," +
+            "    {'INDEX.sizeInGB': 100, core : r5}," +
+            "    {'INDEX.sizeInGB': 100, core : r6}]}";
+
+    String autoScalingjson = "{cluster-policy:[" +
+        "    { replica : '<3' , shard : '#EACH', sysprop.zone: [east,west] } ]," +
+        "  'cluster-preferences':[{ minimize : cores},{minimize : freedisk, precision : 50}]}";
+    Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
+    Policy.Session session = policy.createSession(createCloudManager(state, metaData));
+    List<Violation> violations = session.getViolations();
+    assertEquals(1, violations.size());
+    assertEquals(4, violations.get(0).getViolatingReplicas().size());
+    assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01);
+    for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
+      assertEquals("shard2", r.replicaInfo.getShard());
+    }
+
+    autoScalingjson = "{cluster-policy:[" +
+        "    { replica : '<3' , shard : '#EACH', sysprop.zone: '#EACH' } ]," +
+        "  'cluster-preferences':[{ minimize : cores},{minimize : freedisk, precision : 50}]}";
+    policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
+    session = policy.createSession(createCloudManager(state, metaData));
+    violations = session.getViolations();
+    assertEquals(1, violations.size());
+    assertEquals(4, violations.get(0).getViolatingReplicas().size());
+    assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01);
+    for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) {
+      assertEquals("shard2", r.replicaInfo.getShard());
+    }
+
+  }
+
+  static SolrCloudManager createCloudManager(String clusterStateStr, String metadata) {
+    Map m = (Map) Utils.fromJSONString(clusterStateStr);
+    Map meta = (Map) Utils.fromJSONString(metadata);
+    Map nodeVals = (Map) meta.get("nodeValues");
+    List<Map> replicaVals = (List<Map>) meta.get("replicaValues");
+    ClusterState clusterState = ClusterState.load(0, m, Collections.emptySet(), null);
+    Map<String, AtomicInteger> coreCount = new LinkedHashMap<>();
+    Set<String> nodes = new HashSet<>(nodeVals.keySet());
+    clusterState.getCollectionStates().forEach((s, collectionRef) -> collectionRef.get()
+        .forEachReplica((s12, replica) -> {
+          nodes.add(replica.getNodeName());
+          coreCount.computeIfAbsent(replica.getNodeName(), s1 -> new AtomicInteger(0))
+              .incrementAndGet();
+        }));
+
+    DelegatingClusterStateProvider delegatingClusterStateProvider = new DelegatingClusterStateProvider(null) {
+      @Override
+      public ClusterState getClusterState() throws IOException {
+        return clusterState;
+      }
+
+      @Override
+      public Set<String> getLiveNodes() {
+        return nodes;
+      }
+    };
+
+    return new DelegatingCloudManager(null) {
+
+      @Override
+      public ClusterStateProvider getClusterStateProvider() {
+        return delegatingClusterStateProvider;
+      }
+
+      @Override
+      public NodeStateProvider getNodeStateProvider() {
+
+        return new SolrClientNodeStateProvider(null) {
+          @Override
+          protected ClusterStateProvider getClusterStateProvider() {
+            return delegatingClusterStateProvider;
+          }
+
+          @Override
+          protected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {
+            Map<String, Object> result = new LinkedHashMap<>();
+            for (String tag : tags) {
+              if (tag.equals(CORES.tagName))
+                result.put(CORES.tagName, coreCount.getOrDefault(node, new AtomicInteger(0)).get());
+              result.put(tag, Utils.getObjectByPath(nodeVals, true, Arrays.asList(node, tag)));
+            }
+            return result;
+          }
+
+          @Override
+          public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+            Map<String, Map<String, List<ReplicaInfo>>> result = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> emptyMap());
+            if (!keys.isEmpty()) {
+              Row.forEachReplica(result, replicaInfo -> {
+                for (String key : keys) {
+                  if (!replicaInfo.getVariables().containsKey(key)) {
+                    replicaVals.stream()
+                        .filter(it -> replicaInfo.getCore().equals(it.get("core")))
+                        .findFirst()
+                        .ifPresent(map -> replicaInfo.getVariables().put(key, map.get(key)));
+                  }
+                }
+              });
+            }
+            return result;
+          }
+        };
+      }
+    };
+  }
+
+}


Mime
View raw message