lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject [1/2] lucene-solr:master: SOLR-12601: Refactor the autoscaling package to improve readability
Date Tue, 31 Jul 2018 03:12:46 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master a08eadb48 -> 4602e4de4


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
new file mode 100644
index 0000000..8df74bf
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+
+import static java.util.Collections.emptySet;
+import static java.util.Collections.unmodifiableSet;
+
+/**
+ * A Variable Type used in Autoscaling policy rules. Each variable type may have unique implementation
+ * of functionalities
+ */
+public interface Variable {
+  String NULL = "";
+  String coreidxsize = "INDEX.sizeInGB";
+
+  default boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+    return op.match(val, validate(name, inputVal, false)) == Clause.TestStatus.PASS;
+  }
+  default Object convertVal(Object val) {
+    return val;
+  }
+
+  default void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector, boolean strictMode) {
+  }
+
+  default void addViolatingReplicas(Violation.Ctx ctx) {
+    for (Row row : ctx.allRows) {
+      if (ctx.clause.tag.varType.meta.isNodeSpecificVal() && !row.node.equals(ctx.tagKey))
continue;
+      Violation.collectViolatingReplicas(ctx, row);
+    }
+  }
+
+  void getSuggestions(Suggestion.Ctx ctx) ;
+
+  default Object computeValue(Policy.Session session, Clause.Condition condition, String
collection, String shard, String node) {
+    return condition.val;
+  }
+
+  int compareViolation(Violation v1, Violation v2);
+
+  default void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector) {
+  }
+
+  default String postValidate(Clause.Condition condition) {
+    return null;
+  }
+
+  default Operand getOperand(Operand expected, Object strVal, Clause.ComputedType computedType)
{
+    return expected;
+  }
+
+  Object validate(String name, Object val, boolean isRuleVal);
+
+  /**
+   * Type details of each variable in policies
+   */
+  public enum Type implements Variable {
+    @Meta(name = "withCollection", type = String.class, isNodeSpecificVal = true, implementation
= WithCollectionVariable.class)
+    WITH_COLLECTION(),
+
+    @Meta(name = "collection",
+        type = String.class)
+    COLL(),
+    @Meta(
+        name = "shard",
+        type = String.class,
+        wildCards = {Policy.EACH, Policy.ANY})
+    SHARD(),
+
+    @Meta(name = "replica",
+        type = Double.class,
+        min = 0, max = -1,
+        implementation = ReplicaVariable.class,
+        computedValues = {Clause.ComputedType.EQUAL, Clause.ComputedType.PERCENT, Clause.ComputedType.ALL})
+    REPLICA(),
+    @Meta(name = ImplicitSnitch.PORT,
+        type = Long.class,
+        min = 1,
+        max = 65535,
+        supportArrayVals = true,
+        wildCards = Policy.EACH
+    )
+    PORT(),
+    @Meta(name = "ip_1",
+        type = Long.class,
+        min = 0,
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
+    IP_1(),
+    @Meta(name = "ip_2",
+        type = Long.class,
+        min = 0,
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
+    IP_2(),
+    @Meta(name = "ip_3",
+        type = Long.class,
+        min = 0,
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
+    IP_3(),
+    @Meta(name = "ip_4",
+        type = Long.class,
+        min = 0,
+        max = 255,
+        supportArrayVals = true,
+        wildCards = Policy.EACH)
+    IP_4(),
+    @Meta(name = ImplicitSnitch.DISK,
+        type = Double.class,
+        min = 0,
+        isNodeSpecificVal = true,
+        associatedPerReplicaValue = Variable.coreidxsize,
+        associatedPerNodeValue = "totaldisk",
+        implementation = FreeDiskVariable.class,
+        computedValues = Clause.ComputedType.PERCENT)
+    FREEDISK(),
+
+    @Meta(name = "totaldisk",
+        type = Double.class,
+        isHidden = true, implementation = VariableBase.TotalDiskVariable.class)
+    TOTALDISK(),
+
+    @Meta(name = Variable.coreidxsize,
+        type = Double.class,
+        isNodeSpecificVal = true,
+        isHidden = true,
+        min = 0,
+        implementation = VariableBase.CoreIndexSizeVariable.class,
+        metricsKey = "INDEX.sizeInBytes")
+    CORE_IDX(),
+    @Meta(name = ImplicitSnitch.NODEROLE,
+        type = String.class,
+        enumVals = "overseer")
+    NODE_ROLE(),
+
+    @Meta(name = ImplicitSnitch.CORES,
+        type = Long.class,
+        min = 0,
+        implementation = CoresVariable.class)
+    CORES(),
+
+    @Meta(name = ImplicitSnitch.SYSLOADAVG,
+        type = Double.class,
+        min = 0,
+        max = 100,
+        isNodeSpecificVal = true)
+    SYSLOADAVG(),
+
+    @Meta(name = ImplicitSnitch.HEAPUSAGE,
+        type = Double.class,
+        min = 0,
+        isNodeSpecificVal = true)
+    HEAPUSAGE(),
+    @Meta(name = "NUMBER",
+        type = Long.class,
+        min = 0)
+    NUMBER(),
+
+    @Meta(name = "STRING",
+        type = String.class,
+        wildCards = Policy.EACH,
+        supportArrayVals = true)
+    STRING(),
+
+    @Meta(name = "node",
+        type = String.class,
+        isNodeSpecificVal = true,
+        wildCards = {Policy.ANY, Policy.EACH},
+        implementation = NodeVariable.class,
+        supportArrayVals = true)
+    NODE(),
+
+    @Meta(name = "LAZY",
+        type = void.class,
+        implementation = VariableBase.LazyVariable.class)
+    LAZY(),
+
+    @Meta(name = ImplicitSnitch.DISKTYPE,
+        type = String.class,
+        enumVals = {"ssd", "rotational"},
+        implementation = VariableBase.DiskTypeVariable.class,
+        supportArrayVals = true)
+    DISKTYPE();
+
+    public final String tagName;
+    public final Class type;
+    public Meta meta;
+
+    public final Set<String> vals;
+    public final Number min;
+    public final Number max;
+    public final Boolean additive;
+    public final Set<String> wildCards;
+    public final String perReplicaValue;
+    public final Set<String> associatedPerNodeValues;
+    public final String metricsAttribute;
+    public final Set<Clause.ComputedType> supportedComputedTypes;
+    final Variable impl;
+
+
+    Type() {
+      try {
+        meta = Type.class.getField(name()).getAnnotation(Meta.class);
+        if (meta == null) {
+          throw new RuntimeException("Invalid type, should have a @Meta annotation " + name());
+        }
+      } catch (NoSuchFieldException e) {
+        //cannot happen
+      }
+      impl= VariableBase.loadImpl(meta, this);
+
+      this.tagName = meta.name();
+      this.type = meta.type();
+
+      this.vals = readSet(meta.enumVals());
+      this.max = readNum(meta.max());
+      this.min = readNum(meta.min());
+      this.perReplicaValue = readStr(meta.associatedPerReplicaValue());
+      this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue());
+      this.additive = meta.isAdditive();
+      this.metricsAttribute = readStr(meta.metricsKey());
+      this.supportedComputedTypes = meta.computedValues()[0] == Clause.ComputedType.NULL
?
+          emptySet() :
+          unmodifiableSet(new HashSet(Arrays.asList(meta.computedValues())));
+      this.wildCards = readSet(meta.wildCards());
+
+    }
+
+    public String getTagName() {
+      return meta.name();
+    }
+
+    private String readStr(String s) {
+      return NULL.equals(s) ? null : s;
+    }
+
+    private Number readNum(double v) {
+      return v == -1 ? null :
+          (Number) validate(null, v, true);
+    }
+
+    Set<String> readSet(String[] vals) {
+      if (NULL.equals(vals[0])) return emptySet();
+      return unmodifiableSet(new HashSet<>(Arrays.asList(vals)));
+    }
+
+    @Override
+    public void getSuggestions(Suggestion.Ctx ctx) {
+      impl.getSuggestions(ctx);
+    }
+
+    @Override
+    public void addViolatingReplicas(Violation.Ctx ctx) {
+        impl.addViolatingReplicas(ctx);
+    }
+
+    public Operand getOperand(Operand expected, Object val, Clause.ComputedType computedType)
{
+      return impl.getOperand(expected, val, computedType);
+    }
+
+
+    public Object convertVal(Object val) {
+      return impl.convertVal(val);
+    }
+
+    public String postValidate(Clause.Condition condition) {
+      return impl.postValidate(condition);
+    }
+
+    public Object validate(String name, Object val, boolean isRuleVal) {
+      return impl.validate(name, val, isRuleVal);
+    }
+
+    /**
+     * Simulate a replica addition to a node in the cluster
+     */
+    public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector, boolean strictMode) {
+      impl.projectAddReplica(cell, ri, opCollector, strictMode);
+    }
+
+    public void projectRemoveReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector) {
+      impl.projectRemoveReplica(cell, ri, opCollector);
+    }
+
+    @Override
+    public int compareViolation(Violation v1, Violation v2) {
+      return impl.compareViolation(v1, v2);
+    }
+
+    @Override
+    public Object computeValue(Policy.Session session, Clause.Condition condition, String
collection, String shard, String node) {
+      return impl.computeValue(session, condition, collection, shard, node);
+    }
+
+    @Override
+    public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+      return impl.match(inputVal, op, val, name, row);
+    }
+  }
+
+  @Target(ElementType.FIELD)
+  @Retention(RetentionPolicy.RUNTIME)
+  @interface Meta {
+    String name();
+
+    Class type();
+
+    String[] associatedPerNodeValue() default NULL;
+
+    String associatedPerReplicaValue() default NULL;
+
+    String[] enumVals() default NULL;
+
+    String[] wildCards() default NULL;
+
+    boolean isNodeSpecificVal() default false;
+
+    boolean isHidden() default false;
+
+    boolean isAdditive() default true;
+
+    double min() default -1d;
+
+    double max() default -1d;
+
+    boolean supportArrayVals() default false;
+
+    String metricsKey() default NULL;
+
+    Class implementation() default void.class;
+
+    Clause.ComputedType[] computedValues() default Clause.ComputedType.NULL;
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
new file mode 100644
index 0000000..ad2b43b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.util.StrUtils;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.parseString;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.perNodeSuggestions;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+
+public class VariableBase implements Variable {
+  final Type varType;
+
+  public VariableBase(Type type) {
+    this.varType = type;
+  }
+
+  @Override
+  public void getSuggestions(Suggestion.Ctx ctx) {
+    perNodeSuggestions(ctx);
+  }
+
+  static Object getOperandAdjustedValue(Object val, Object original) {
+    if (original instanceof Clause.Condition) {
+      Clause.Condition condition = (Clause.Condition) original;
+      if (condition.computedType == null && isIntegerEquivalent(val)) {
+        if (condition.op == Operand.LESS_THAN) {
+          //replica : '<3'
+          val = val instanceof Long ?
+              (Long) val - 1 :
+              (Double) val - 1;
+        } else if (condition.op == Operand.GREATER_THAN) {
+          //replica : '>4'
+          val = val instanceof Long ?
+              (Long) val + 1 :
+              (Double) val + 1;
+        }
+      }
+    }
+    return val;
+  }
+
+  static boolean isIntegerEquivalent(Object val) {
+    if (val instanceof Number) {
+      Number number = (Number) val;
+      return Math.ceil(number.doubleValue()) == Math.floor(number.doubleValue());
+    } else if (val instanceof String) {
+      try {
+        double dval = Double.parseDouble((String) val);
+        return Math.ceil(dval) == Math.floor(dval);
+      } catch (NumberFormatException e) {
+        return false;
+      }
+    } else {
+      return false;
+    }
+
+  }
+
+  public static Type getTagType(String name) {
+    Type info = validatetypes.get(name);
+    if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = Type.STRING;
+    if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = Type.LAZY;
+    return info;
+  }
+
+  static Variable loadImpl(Meta meta, Type t) {
+    Class implementation = meta.implementation();
+    if (implementation == void.class) implementation = VariableBase.class;
+    try {
+      return (Variable) implementation.getConstructor(Type.class).newInstance(t);
+    } catch (Exception e) {
+      throw new RuntimeException("Unable to instantiate: " + implementation.getName(), e);
+    }
+  }
+
+  @Override
+  public int compareViolation(Violation v1, Violation v2) {
+    if (v2.replicaCountDelta == null || v1.replicaCountDelta == null) return 0;
+    if (Math.abs(v1.replicaCountDelta) == Math.abs(v2.replicaCountDelta)) return 0;
+    return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1;
+  }
+
+  @Override
+  public Object validate(String name, Object val, boolean isRuleVal) {
+    if (val instanceof Clause.Condition) {
+      Clause.Condition condition = (Clause.Condition) val;
+      val = condition.op.readRuleValue(condition);
+      if (val != condition.val) return val;
+    }
+    if (name == null) name = this.varType.tagName;
+    if (varType.type == Double.class) {
+      Double num = Clause.parseDouble(name, val);
+      if (isRuleVal) {
+        if (varType.min != null)
+          if (Double.compare(num, varType.min.doubleValue()) == -1)
+            throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min);
+        if (varType.max != null)
+          if (Double.compare(num, varType.max.doubleValue()) == 1)
+            throw new RuntimeException(name + ": " + val + " must be less than " + varType.max);
+      }
+      return num;
+    } else if (varType.type == Long.class) {
+      Long num = Clause.parseLong(name, val);
+      if (isRuleVal) {
+        if (varType.min != null)
+          if (num < varType.min.longValue())
+            throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min);
+        if (varType.max != null)
+          if (num > varType.max.longValue())
+            throw new RuntimeException(name + ": " + val + " must be less than " + varType.max);
+      }
+      return num;
+    } else if (varType.type == String.class) {
+      if (isRuleVal && !varType.vals.isEmpty() && !varType.vals.contains(val))
+        throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(varType.vals,
','));
+      return val;
+    } else {
+      throw new RuntimeException("Invalid type ");
+    }
+  }
+
+  public static class TotalDiskVariable extends VariableBase {
+    public TotalDiskVariable(Type type) {
+      super(type);
+    }
+
+    @Override
+    public Object convertVal(Object val) {
+      return FREEDISK.convertVal(val);
+    }
+  }
+
+  public static class CoreIndexSizeVariable extends VariableBase {
+    public CoreIndexSizeVariable(Type type) {
+      super(type);
+    }
+
+    @Override
+    public Object convertVal(Object val) {
+      return FREEDISK.convertVal(val);
+    }
+  }
+
+  public static class LazyVariable extends VariableBase {
+    public LazyVariable(Type type) {
+      super(type);
+    }
+
+    @Override
+    public Object validate(String name, Object val, boolean isRuleVal) {
+      return parseString(val);
+    }
+
+    @Override
+    public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+      return op.match(parseString(val), parseString(inputVal)) == Clause.TestStatus.PASS;
+    }
+
+    @Override
+    public void getSuggestions(Suggestion.Ctx ctx) {
+      perNodeSuggestions(ctx);
+    }
+  }
+
+  public static class DiskTypeVariable extends VariableBase {
+    public DiskTypeVariable(Type type) {
+      super(type);
+    }
+
+    @Override
+    public void getSuggestions(Suggestion.Ctx ctx) {
+      perNodeSuggestions(ctx);
+    }
+
+
+  }
+
+  private static Map<String, Type> validatetypes;
+
+  static {
+    validatetypes = new HashMap<>();
+    for (Type t : Type.values())
+      validatetypes.put(t.tagName, t);
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 7b0f0f3..2f81291 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Function;
 
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.util.Utils;
@@ -47,6 +48,29 @@ public class Violation implements MapWriter {
     hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new
HashMap<>()))).hashCode();
   }
 
+  static void collectViolatingReplicas(Ctx ctx, Row row) {
+    if (ctx.clause.tag.varType.meta.isNodeSpecificVal()) {
+      row.forEachReplica(replica -> {
+        if (ctx.clause.collection.isPass(replica.getCollection()) && ctx.clause.getShard().isPass(replica.getShard()))
{
+          ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica)
+              .withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+        }
+      });
+    } else {
+      row.forEachReplica(replica -> {
+        if (ctx.clause.replica.isPass(0) && !ctx.clause.tag.isPass(row)) return;
+        if (!ctx.clause.replica.isPass(0) && ctx.clause.tag.isPass(row)) return;
+        if(!ctx.currentViolation.getClause().matchShard(replica.getShard(), ctx.currentViolation.shard))
return;
+        if (!ctx.clause.collection.isPass(ctx.currentViolation.coll) || !ctx.clause.shard.isPass(ctx.currentViolation.shard))
+          return;
+        ctx.currentViolation.addReplica(new ReplicaInfoAndErr(replica).withDelta(ctx.clause.tag.delta(row.getVal(ctx.clause.tag.name))));
+      });
+
+    }
+
+
+  }
+
   public Violation addReplica(ReplicaInfoAndErr r) {
     replicaInfoAndErrs.add(r);
     return this;
@@ -144,4 +168,29 @@ public class Violation implements MapWriter {
     });
     ew.put("clause", getClause());
   }
+
+  static class Ctx {
+    final Function<Clause.Condition, Object> evaluator;
+    String tagKey;
+    Clause clause;
+    ReplicaCount count;
+    Violation currentViolation;
+    List<Row> allRows;
+    List<Violation> allViolations = new ArrayList<>();
+
+    public Ctx(Clause clause, List<Row> allRows, Function<Clause.Condition, Object>
evaluator) {
+      this.allRows = allRows;
+      this.clause = clause;
+      this.evaluator = evaluator;
+    }
+
+    public Ctx reset(String tagKey, ReplicaCount count, Violation currentViolation) {
+      this.tagKey = tagKey;
+      this.count = count;
+      this.currentViolation = currentViolation;
+      allViolations.add(currentViolation);
+      this.clause = currentViolation.getClause();
+      return this;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
deleted file mode 100644
index 989a087..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVarType.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.cloud.autoscaling;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Consumer;
-
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.util.Pair;
-
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-
-/**
- * Implements the 'withCollection' variable type
- */
-public class WithCollectionVarType implements VarType {
-  @Override
-  public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
-    Map<String, String> withCollectionMap = (Map<String, String>) inputVal;
-    if (withCollectionMap == null || withCollectionMap.isEmpty()) return true;
-
-    Set<String> uniqueColls = new HashSet<>();
-    row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
-
-    for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
-      if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue()))
return false;
-    }
-
-    return true;
-  }
-
-  public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector, boolean strictMode) {
-    if (strictMode) {
-      // we do not want to add a replica of the 'withCollection' in strict mode
-      return;
-    }
-
-    Map<String, String> withCollectionMap = (Map<String, String>) cell.val;
-    if (withCollectionMap == null || withCollectionMap.isEmpty()) return;
-
-    Set<String> uniqueColls = new HashSet<>();
-    Row row = cell.row;
-    row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
-
-    for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
-      if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue()))
{
-        String withCollection = e.getValue();
-
-        opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name,
true, Replica.Type.NRT));
-      }
-    }
-  }
-
-  @Override
-  public int compareViolation(Violation v1, Violation v2) {
-    return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size());
-  }
-
-  public void addViolatingReplicas(Suggestion.ViolationCtx ctx) {
-    String node = ctx.currentViolation.node;
-    for (Row row : ctx.allRows) {
-      if (node.equals(row.node)) {
-        Map<String, String> withCollectionMap = (Map<String, String>) row.getVal("withCollection");
-        if (withCollectionMap != null) {
-          row.forEachReplica(r -> {
-            String withCollection = withCollectionMap.get(r.getCollection());
-            if (withCollection != null) {
-              // test whether this row has at least 1 replica of withCollection, else there
is a violation
-              Set<String> uniqueCollections = new HashSet<>();
-              row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection()));
-              if (!uniqueCollections.contains(withCollection)) {
-                ctx.currentViolation.addReplica(new Violation.ReplicaInfoAndErr(r).withDelta(1.0d));
-              }
-            }
-          });
-          ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size();
-        }
-      }
-    }
-  }
-
-  @Override
-  public void getSuggestions(Suggestion.SuggestionCtx ctx) {
-    if (ctx.violation.getViolatingReplicas().isEmpty()) return;
-
-    Map<String, Object> nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node,
Collections.singleton("withCollection"));
-    Map<String, String> withCollectionsMap = (Map<String, String>) nodeValues.get("withCollection");
-    if (withCollectionsMap == null) return;
-
-    Set<String> uniqueCollections = new HashSet<>();
-    for (Violation.ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas())
{
-      uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection());
-    }
-
-    collectionLoop:
-    for (String collection : uniqueCollections) {
-      String withCollection = withCollectionsMap.get(collection);
-      if (withCollection == null) continue;
-
-      // can we find a node from which we can move a replica of the `withCollection`
-      // without creating another violation?
-      for (Row row : ctx.session.matrix) {
-        if (ctx.violation.node.equals(row.node))  continue; // filter the violating node
-
-        Set<String> hostedCollections = new HashSet<>();
-        row.forEachReplica(replicaInfo -> hostedCollections.add(replicaInfo.getCollection()));
-
-        if (hostedCollections.contains(withCollection) && !hostedCollections.contains(collection))
 {
-          // find the candidate replicas that we can move
-          List<ReplicaInfo> movableReplicas = new ArrayList<>();
-          row.forEachReplica(replicaInfo -> {
-            if (replicaInfo.getCollection().equals(withCollection)) {
-              movableReplicas.add(replicaInfo);
-            }
-          });
-
-          for (ReplicaInfo toMove : movableReplicas) {
-            // candidate source node for a move replica operation
-            Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
-                .forceOperation(true)
-                .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
-                .hint(Suggester.Hint.SRC_NODE, row.node)
-                .hint(Suggester.Hint.REPLICA, toMove.getName())
-                .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
-            if (ctx.addSuggestion(suggester) != null)
-              continue collectionLoop; // one suggestion is enough for this collection
-          }
-        }
-      }
-
-      // we could not find a valid move, so we suggest adding a replica
-      Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
-          .forceOperation(true)
-          .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
-          .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
-      ctx.addSuggestion(suggester);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
new file mode 100644
index 0000000..b295aee
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Consumer;
+
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.util.Pair;
+
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+
+/**
+ * Implements the 'withCollection' variable type
+ */
+public class WithCollectionVariable extends VariableBase {
+
+  public WithCollectionVariable(Type type) {
+    super(type);
+  }
+
+  @Override
+  public boolean match(Object inputVal, Operand op, Object val, String name, Row row) {
+    Map<String, String> withCollectionMap = (Map<String, String>) inputVal;
+    if (withCollectionMap == null || withCollectionMap.isEmpty()) return true;
+
+    Set<String> uniqueColls = new HashSet<>();
+    row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
+
+    for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
+      if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue()))
return false;
+    }
+
+    return true;
+  }
+
+  public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo>
opCollector, boolean strictMode) {
+    if (strictMode) {
+      // we do not want to add a replica of the 'withCollection' in strict mode
+      return;
+    }
+
+    Map<String, String> withCollectionMap = (Map<String, String>) cell.val;
+    if (withCollectionMap == null || withCollectionMap.isEmpty()) return;
+
+    Set<String> uniqueColls = new HashSet<>();
+    Row row = cell.row;
+    row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));
+
+    for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
+      if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue()))
{
+        String withCollection = e.getValue();
+
+        opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name,
true, Replica.Type.NRT));
+      }
+    }
+  }
+
+  @Override
+  public int compareViolation(Violation v1, Violation v2) {
+    return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size());
+  }
+
+  public void addViolatingReplicas(Violation.Ctx ctx) {
+    String node = ctx.currentViolation.node;
+    for (Row row : ctx.allRows) {
+      if (node.equals(row.node)) {
+        Map<String, String> withCollectionMap = (Map<String, String>) row.getVal("withCollection");
+        if (withCollectionMap != null) {
+          row.forEachReplica(r -> {
+            String withCollection = withCollectionMap.get(r.getCollection());
+            if (withCollection != null) {
+              // test whether this row has at least 1 replica of withCollection, else there
is a violation
+              Set<String> uniqueCollections = new HashSet<>();
+              row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection()));
+              if (!uniqueCollections.contains(withCollection)) {
+                ctx.currentViolation.addReplica(new Violation.ReplicaInfoAndErr(r).withDelta(1.0d));
+              }
+            }
+          });
+          ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size();
+        }
+      }
+    }
+  }
+
+  @Override
+  public void getSuggestions(Suggestion.Ctx ctx) {
+    if (ctx.violation.getViolatingReplicas().isEmpty()) return;
+
+    Map<String, Object> nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node,
Collections.singleton("withCollection"));
+    Map<String, String> withCollectionsMap = (Map<String, String>) nodeValues.get("withCollection");
+    if (withCollectionsMap == null) return;
+
+    Set<String> uniqueCollections = new HashSet<>();
+    for (Violation.ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas())
{
+      uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection());
+    }
+
+    collectionLoop:
+    for (String collection : uniqueCollections) {
+      String withCollection = withCollectionsMap.get(collection);
+      if (withCollection == null) continue;
+
+      // can we find a node from which we can move a replica of the `withCollection`
+      // without creating another violation?
+      for (Row row : ctx.session.matrix) {
+        if (ctx.violation.node.equals(row.node))  continue; // filter the violating node
+
+        Set<String> hostedCollections = new HashSet<>();
+        row.forEachReplica(replicaInfo -> hostedCollections.add(replicaInfo.getCollection()));
+
+        if (hostedCollections.contains(withCollection) && !hostedCollections.contains(collection))
 {
+          // find the candidate replicas that we can move
+          List<ReplicaInfo> movableReplicas = new ArrayList<>();
+          row.forEachReplica(replicaInfo -> {
+            if (replicaInfo.getCollection().equals(withCollection)) {
+              movableReplicas.add(replicaInfo);
+            }
+          });
+
+          for (ReplicaInfo toMove : movableReplicas) {
+            // candidate source node for a move replica operation
+            Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+                .forceOperation(true)
+                .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
+                .hint(Suggester.Hint.SRC_NODE, row.node)
+                .hint(Suggester.Hint.REPLICA, toMove.getName())
+                .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
+            if (ctx.addSuggestion(suggester) != null)
+              continue collectionLoop; // one suggestion is enough for this collection
+          }
+        }
+      }
+
+      // we could not find a valid move, so we suggest adding a replica
+      Suggester suggester = ctx.session.getSuggester(ADDREPLICA)
+          .forceOperation(true)
+          .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1"))
+          .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node);
+      ctx.addSuggestion(suggester);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 2015b52..83fb25a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -36,7 +36,8 @@ import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.cloud.NodeStateProvider;
 import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
 import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion;
+import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
+import org.apache.solr.client.solrj.cloud.autoscaling.VariableBase;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.response.SimpleSolrResponse;
 import org.apache.solr.common.MapWriter;
@@ -59,9 +60,9 @@ import org.slf4j.LoggerFactory;
 
 import static java.util.Collections.emptyMap;
 import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.TOTALDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.WITH_COLLECTION;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.WITH_COLLECTION;
 
 /**
  *
@@ -153,7 +154,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider,
MapWriter
         for (String key : keys) {
           if (r.getVariables().containsKey(key)) continue;// it's already collected
           String perReplicaMetricsKey = "solr.core." + r.getCollection() + "." + r.getShard()
+ "." + Utils.parseMetricsReplicaName(r.getCollection(), r.getCore()) + ":";
-          Suggestion.ConditionType tagType = Suggestion.getTagType(key);
+          Type tagType = VariableBase.getTagType(key);
           String perReplicaValue = key;
           if (tagType != null) {
             perReplicaValue = tagType.metricsAttribute;
@@ -168,7 +169,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider,
MapWriter
         Map<String, Object> tagValues = fetchReplicaMetrics(node, metricsKeyVsTagReplica);
         tagValues.forEach((k, o) -> {
           Pair<String, ReplicaInfo> p = metricsKeyVsTagReplica.get(k);
-          Suggestion.ConditionType validator = Suggestion.getTagType(p.first());
+          Type validator = VariableBase.getTagType(p.first());
           if (validator != null) o = validator.convertVal(o);
           if (p.second() != null) p.second().getVariables().put(p.first(), o);
         });

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 16addd4..16dfdcd 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -43,7 +43,6 @@ import org.apache.solr.client.solrj.cloud.DistribStateManager;
 import org.apache.solr.client.solrj.cloud.DistributedQueueFactory;
 import org.apache.solr.client.solrj.cloud.NodeStateProvider;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.Clause.RangeVal;
 import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
 import org.apache.solr.client.solrj.impl.ClusterStateProvider;
 import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
@@ -72,9 +71,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.REPLICA;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.REPLICA;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
 
@@ -3413,18 +3412,18 @@ public void testUtilizeNodeFailure2() throws Exception {
         "        {'node':'solr-27:8983_solr'}]}]}";
 
     List l = (List) ((Map) Utils.fromJSONString(rowsData)).get("sortedNodes");
-    List<Suggestion.ConditionType> params = new ArrayList<>();
+    List<Variable.Type> params = new ArrayList<>();
     params.add(CORES);
-    params.add(Suggestion.ConditionType.FREEDISK);
-    params.add(Suggestion.ConditionType.SYSLOADAVG);
-    params.add(Suggestion.ConditionType.NODE);
+    params.add(Variable.Type.FREEDISK);
+    params.add(Variable.Type.SYSLOADAVG);
+    params.add(Variable.Type.NODE);
     List<Row> rows = new ArrayList<>();
     for (Object o : l) {
       Map m = (Map) o;
       Cell[] c = new Cell[params.size()];
       List attrs = (List) m.get("attributes");
       for (int i = 0; i < params.size(); i++) {
-        Suggestion.ConditionType param = params.get(i);
+        Variable.Type param = params.get(i);
         for (Object attr : attrs) {
           Object o1 = ((Map) attr).get(param.tagName);
           if (o1 != null) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4602e4de/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 9c5528a..678600f 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -40,7 +40,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static java.util.Collections.emptyMap;
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.CORES;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
 
 public class TestPolicy2 extends SolrTestCaseJ4 {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());


Mime
View raw message