lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject lucene-solr:jira/solr-11285-sim: SOLR-11285: Add some javadocs. Try to test searchRate trigger (still broken).
Date Fri, 01 Dec 2017 15:28:37 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-11285-sim d27dd2105 -> 5d2524e9b


SOLR-11285: Add some javadocs. Try to test searchRate trigger (still broken).


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5d2524e9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5d2524e9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5d2524e9

Branch: refs/heads/jira/solr-11285-sim
Commit: 5d2524e9b198ee062c8f9cde711889af190b885c
Parents: d27dd21
Author: Andrzej Bialecki <ab@apache.org>
Authored: Fri Dec 1 16:28:07 2017 +0100
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Fri Dec 1 16:28:07 2017 +0100

----------------------------------------------------------------------
 .../cloud/autoscaling/ComputePlanAction.java    |  12 +-
 .../cloud/autoscaling/SearchRateTrigger.java    |   5 +-
 .../autoscaling/TriggerIntegrationTest.java     |   7 +
 .../solr/cloud/autoscaling/sim/ActionError.java |   2 +-
 .../sim/GenericDistributedQueue.java            |   2 +-
 .../cloud/autoscaling/sim/SimCloudManager.java  |  46 ++++-
 .../sim/SimClusterStateProvider.java            | 207 +++++++++++++++++--
 .../sim/SimDistributedQueueFactory.java         |   4 +-
 .../autoscaling/sim/SimNodeStateProvider.java   |  45 +++-
 .../cloud/autoscaling/sim/TestLargeCluster.java |  62 +++++-
 10 files changed, 361 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 9eed1f2..c770cac 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -35,6 +35,7 @@ import org.apache.solr.common.params.AutoScalingParams;
 import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudManager;
 import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
 import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.util.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -112,13 +113,10 @@ public class ComputePlanAction extends TriggerActionBase {
         } else {
           // collection || shard || replica -> ADDREPLICA
           suggester = session.getSuggester(CollectionParams.CollectionAction.ADDREPLICA);
-          Set<String> collections = new HashSet<>();
-          // XXX improve this when AddReplicaSuggester supports coll_shard hint
-          hotReplicas.forEach(r -> collections.add(r.getCollection()));
-          hotShards.forEach((coll, shards) -> collections.add(coll));
-          hotCollections.forEach((coll, rate) -> collections.add(coll));
-          for (String coll : collections) {
-            suggester = suggester.hint(Suggester.Hint.COLL, coll);
+          Set<Pair> collectionShards = new HashSet<>();
+          hotShards.forEach((coll, shards) -> shards.forEach((s, r) -> collectionShards.add(new
Pair(coll, s))));
+          for (Pair<String, String> colShard : collectionShards) {
+            suggester = suggester.hint(Suggester.Hint.COLL_SHARD, colShard);
           }
         }
         break;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index 1e0caec..ec3110e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -165,6 +165,9 @@ public class SearchRateTrigger extends TriggerBase {
           });
         });
       });
+      if (metricTags.isEmpty()) {
+        continue;
+      }
       Map<String, Object> rates = cloudManager.getNodeStateProvider().getNodeValues(node,
metricTags.keySet());
       rates.forEach((tag, rate) -> {
         ReplicaInfo info = metricTags.get(tag);
@@ -271,7 +274,7 @@ public class SearchRateTrigger extends TriggerBase {
   private boolean waitForElapsed(String name, long now, Map<String, Long> lastEventMap)
{
     Long lastTime = lastEventMap.computeIfAbsent(name, s -> now);
     long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS);
-    log.debug("name=" + name + ", lastTime=" + lastTime + ", elapsed=" + elapsed);
+    log.trace("name={}, lastTime={}, elapsed={}", name, lastTime, elapsed);
     if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond())
{
       return false;
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index f7e1a62..ebd7cbd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -1394,6 +1394,10 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
 
   @Test
   public void testSearchRate() throws Exception {
+    // start a few more jetty-s
+    for (int i = 0; i < 3; i++) {
+      cluster.startJettySolrRunner();
+    }
     CloudSolrClient solrClient = cluster.getSolrClient();
     String COLL1 = "collection1";
     CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
@@ -1407,6 +1411,8 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
         "'enabled' : true," +
         "'rate' : 1.0," +
         "'actions' : [" +
+        "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+        "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
         "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" +
         "]" +
         "}}";
@@ -1420,6 +1426,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
         "'name' : 'srt'," +
         "'trigger' : 'search_rate_trigger'," +
         "'stage' : ['FAILED','SUCCEEDED']," +
+        "'afterAction': ['compute', 'execute', 'test']," +
         "'class' : '" + TestTriggerListener.class.getName() + "'" +
         "}" +
         "}";

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
index c10b42e..c1c070d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/ActionError.java
@@ -17,7 +17,7 @@
 package org.apache.solr.cloud.autoscaling.sim;
 
 /**
- *
+ * Interface that helps simulating action errors.
  */
 public interface ActionError {
   boolean shouldFail(String... args);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
index ee7e839..5d7aa4d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java
@@ -53,7 +53,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * A distributed queue that uses {@link org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager}.
+ * A distributed queue that uses {@link DistribStateManager} as the underlying distributed
store.
  * Implementation based on {@link org.apache.solr.cloud.ZkDistributedQueue}
  */
 public class GenericDistributedQueue implements DistributedQueue {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index 4a37e35..ecfc8f7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -159,6 +159,11 @@ public class SimCloudManager implements SolrCloudManager {
     return cloudManager;
   }
 
+  /**
+   * Create node values (metrics) for a node.
+   * @param nodeName node name (eg. '127.0.0.1:10000_solr')
+   * @return node values
+   */
   public static Map<String, Object> createNodeValues(String nodeName) {
     Map<String, Object> values = new HashMap<>();
     String host, nodeId;
@@ -206,6 +211,10 @@ public class SimCloudManager implements SolrCloudManager {
     return values;
   }
 
+  /**
+   * Add a new node and initialize its node values (metrics).
+   * @return new node id
+   */
   public String simAddNode() throws Exception {
     Map<String, Object> values = createNodeValues(null);
     String nodeId = (String)values.get(ImplicitSnitch.NODE);
@@ -215,12 +224,21 @@ public class SimCloudManager implements SolrCloudManager {
     return nodeId;
   }
 
+  /**
+   * Remove a node from the cluster. This simulates a node lost scenario.
+   * @param nodeId node id
+   */
   public void simRemoveNode(String nodeId) throws Exception {
     clusterStateProvider.simRemoveNode(nodeId);
     nodeStateProvider.simRemoveNodeValues(nodeId);
     LOG.trace("-- removed node " + nodeId);
   }
 
+  /**
+   * Remove a number of randomly selected nodes
+   * @param number number of nodes to remove
+   * @param random random
+   */
   public void simRemoveRandomNodes(int number, Random random) throws Exception {
     List<String> nodes = new ArrayList<>(liveNodes);
     Collections.shuffle(nodes, random);
@@ -230,14 +248,26 @@ public class SimCloudManager implements SolrCloudManager {
     }
   }
 
+  /**
+   * Clear the (simulated) .system collection.
+   */
   public void simClearSystemCollection() {
     systemColl.clear();
   }
 
+  /**
+   * Get the content of (simulated) .system collection.
+   * @return documents in the collection.
+   */
   public List<SolrInputDocument> simGetSystemCollection() {
     return systemColl;
   }
 
+  /**
+   * Get a {@link SolrClient} implementation where calls are forwarded to this
+   * instance.
+   * @return simulated SolrClient.
+   */
   public SolrClient simGetSolrClient() {
     if (solrClient != null) {
       return solrClient;
@@ -257,6 +287,11 @@ public class SimCloudManager implements SolrCloudManager {
     }
   }
 
+  /**
+   * Submit a task to execute in a thread pool.
+   * @param callable task to execute
+   * @return future to obtain results
+   */
   public <T> Future<T> submit(Callable<T> callable) {
     return simCloudManagerPool.submit(callable);
   }
@@ -325,6 +360,12 @@ public class SimCloudManager implements SolrCloudManager {
     }
   }
 
+  /**
+   * Handler for autoscaling requests. NOTE: only a specific subset of autoscaling requests
is
+   * supported!
+   * @param req autoscaling request
+   * @return results
+   */
   public SolrResponse simHandleSolrRequest(SolrRequest req) throws IOException, InterruptedException
{
     // pay the penalty for remote request, at least 5 ms
     timeSource.sleep(5);
@@ -466,8 +507,9 @@ public class SimCloudManager implements SolrCloudManager {
 
   }
 
-
-
+  /**
+   * HTTP requests are not supported by this implementation.
+   */
   @Override
   public byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String>
headers, String payload, int timeout, boolean followRedirects) throws IOException {
     throw new UnsupportedOperationException("general HTTP requests are not supported yet");

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 1f1788b..f65f31a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -105,8 +105,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
   private AtomicReference<Map<String, DocCollection>> collectionsStatesRef =
new AtomicReference<>();
 
   /**
-   * Zero-arg constructor. The instance needs to be initialized using the <code>sim*</code>
methods in order
-   * to ensure proper behavior, otherwise it will behave as a cluster with zero live nodes
and zero replicas.
+   * The instance needs to be initialized using the <code>sim*</code> methods
in order
+   * to ensure proper behavior, otherwise it will behave as a cluster with zero replicas.
    */
   public SimClusterStateProvider(Set<String> liveNodes, SimCloudManager cloudManager)
{
     this.liveNodes = liveNodes;
@@ -126,6 +126,10 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
 
   // ============== SIMULATOR SETUP METHODS ====================
 
+  /**
+   * Initialize from an existing cluster state
+   * @param initialState initial cluster state
+   */
   public void simSetClusterState(ClusterState initialState) throws Exception {
     lock.lock();
     try {
@@ -154,10 +158,18 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Reset the leader election throttle.
+   */
   public void simResetLeaderThrottle() {
     leaderThrottle.reset();
   }
 
+  /**
+   * Get random node id.
+   * @param random instance of random.
+   * @return one of the live nodes
+   */
   public String simGetRandomNode(Random random) {
     if (liveNodes.isEmpty()) {
       return null;
@@ -168,15 +180,20 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
 
   // todo: maybe hook up DistribStateManager /live_nodes ?
   // todo: maybe hook up DistribStateManager /clusterstate.json watchers?
-  public boolean simAddNode(String nodeId) throws Exception {
+
+  /**
+   * Add a new node to the cluster.
+   * @param nodeId unique node id
+   */
+  public void simAddNode(String nodeId) throws Exception {
     if (liveNodes.contains(nodeId)) {
       throw new Exception("Node " + nodeId + " already exists");
     }
     liveNodes.add(nodeId);
-    return nodeReplicaMap.putIfAbsent(nodeId, new ArrayList<>()) == null;
+    nodeReplicaMap.putIfAbsent(nodeId, new ArrayList<>());
   }
 
-  // utility to run leader election in a separate thread and with throttling
+  // utility class to run leader election in a separate thread and with throttling
   // Note: leader election is a no-op if a shard leader already exists for each shard
   private class LeaderElection implements Callable<Boolean> {
     Collection<String> collections;
@@ -202,6 +219,13 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
 
   // todo: maybe hook up DistribStateManager /live_nodes ?
   // todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+  /**
+   * Remove node from a cluster. This is equivalent to a situation when a node is lost.
+   * All replicas that were assigned to this node are marked as DOWN.
+   * @param nodeId node id
+   * @return true if a node existed and was removed
+   */
   public boolean simRemoveNode(String nodeId) throws Exception {
     lock.lock();
     try {
@@ -224,6 +248,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Add a new replica. Note that if any details of a replica (node, coreNodeName, SolrCore
name, etc)
+   * are missing they will be filled in using the policy framework.
+   * @param message replica details
+   * @param results result of the operation
+   */
   public void simAddReplica(ZkNodeProps message, NamedList results) throws Exception {
     AtomicLong policyVersionAfter = new AtomicLong(-1);
     ClusterState clusterState = getClusterState();
@@ -245,6 +275,14 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
   }
 
   // todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+  /**
+   * Add a replica. Note that all details of the replica must be present here, including
+   * node, coreNodeName and SolrCore name.
+   * @param nodeId node id where the replica will be added
+   * @param replicaInfo replica info
+   * @param runLeaderElection if true then run a leader election after adding the replica.
+   */
   public void simAddReplica(String nodeId, ReplicaInfo replicaInfo, boolean runLeaderElection)
throws Exception {
     // make sure coreNodeName is unique across cluster
     for (Map.Entry<String, List<ReplicaInfo>> e : nodeReplicaMap.entrySet())
{
@@ -306,6 +344,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
   }
 
   // todo: maybe hook up DistribStateManager /clusterstate.json watchers?
+
+  /**
+   * Remove replica.
+   * @param nodeId node id
+   * @param coreNodeName coreNodeName
+   */
   public void simRemoveReplica(String nodeId, String coreNodeName) throws Exception {
     List<ReplicaInfo> replicas = nodeReplicaMap.computeIfAbsent(nodeId, n -> new
ArrayList<>());
     lock.lock();
@@ -334,6 +378,10 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Save clusterstate.json to {@link DistribStateManager}.
+   * @return saved state
+   */
   private ClusterState saveClusterState() throws IOException {
     collectionsStatesRef.set(null);
     ClusterState currentState = getClusterState();
@@ -352,6 +400,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     return currentState;
   }
 
+  /**
+   * Delay an operation by a configured amount.
+   * @param collection collection name
+   * @param op operation name.
+   */
   private void opDelay(String collection, String op) throws InterruptedException {
     Map<String, Long> delays = opDelays.get(collection);
     if (delays == null || delays.isEmpty() || !delays.containsKey(op)) {
@@ -360,6 +413,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     cloudManager.getTimeSource().sleep(delays.get(op));
   }
 
+  /**
+   * Simulate running a shard leader election. This operation is a no-op if a leader already
exists.
+   * If a new leader is elected the cluster state is saved.
+   * @param collections list of affected collections
+   * @param saveClusterState if true then save cluster state regardless of changes.
+   */
   private synchronized void simRunLeaderElection(Collection<String> collections, boolean
saveClusterState) throws Exception {
     ClusterState state = getClusterState();
     AtomicBoolean stateChanged = new AtomicBoolean(Boolean.FALSE);
@@ -433,6 +492,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Create a new collection. This operation uses policy framework for node and replica assignments.
+   * @param props collection details
+   * @param results results of the operation.
+   */
   public void simCreateCollection(ZkNodeProps props, NamedList results) throws Exception
{
     if (props.getStr(CommonAdminParams.ASYNC) != null) {
       results.add(CoreAdminParams.REQUESTID, props.getStr(CommonAdminParams.ASYNC));
@@ -488,6 +552,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     results.add("success", "");
   }
 
+  /**
+   * Delete a collection
+   * @param collection collection name
+   * @param async async id
+   * @param results results of the operation
+   */
   public void simDeleteCollection(String collection, String async, NamedList results) throws
IOException {
     if (async != null) {
       results.add(CoreAdminParams.REQUESTID, async);
@@ -525,6 +595,9 @@ public class SimClusterStateProvider implements ClusterStateProvider {
     }
   }
 
+  /**
+   * Remove all collections.
+   */
   public void simDeleteAllCollections() throws Exception {
     lock.lock();
     try {
@@ -540,6 +613,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Move replica. This uses a similar algorithm as {@link org.apache.solr.cloud.MoveReplicaCmd#moveNormalReplica(ClusterState,
NamedList, String, String, DocCollection, Replica, Slice, int, boolean)}.
+   * @param message operation details
+   * @param results operation results.
+   */
   public void simMoveReplica(ZkNodeProps message, NamedList results) throws Exception {
     if (message.getStr(CommonAdminParams.ASYNC) != null) {
       results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC));
@@ -582,6 +660,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     results.add("success", "");
   }
 
+  /**
+   * Create a new shard. This uses a similar algorithm as {@link CreateShardCmd}.
+   * @param message operation details
+   * @param results operation results
+   */
   public void simCreateShard(ZkNodeProps message, NamedList results) throws Exception {
     if (message.getStr(CommonAdminParams.ASYNC) != null) {
       results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC));
@@ -638,6 +721,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Split a shard. This uses a similar algorithm as {@link SplitShardCmd}, including simulating
its
+   * quirks, and leaving the original parent slice in place.
+   * @param message operation details
+   * @param results operation results.
+   */
   public void simSplitShard(ZkNodeProps message, NamedList results) throws Exception {
     String collectionName = message.getStr(COLLECTION_PROP);
     AtomicReference<String> sliceName = new AtomicReference<>();
@@ -696,6 +785,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
 
   }
 
+  /**
+   * Delete a shard. This uses a similar algorithm as {@link org.apache.solr.cloud.DeleteShardCmd}
+   * @param message operation details
+   * @param results operation results
+   */
   public void simDeleteShard(ZkNodeProps message, NamedList results) throws Exception {
     String collectionName = message.getStr(COLLECTION_PROP);
     String sliceName = message.getStr(SHARD_ID_PROP);
@@ -731,7 +825,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
-  public synchronized Map<String, Object> saveClusterProperties() throws Exception
{
+  /**
+   * Saves cluster properties to clusterprops.json.
+   * @return current properties
+   */
+  private synchronized Map<String, Object> saveClusterProperties() throws Exception
{
     if (lastSavedProperties != null && lastSavedProperties.equals(clusterProperties))
{
       return lastSavedProperties;
     }
@@ -743,6 +841,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     return lastSavedProperties;
   }
 
+  /**
+   * Set all cluster properties. This also updates the clusterprops.json data in
+   * {@link DistribStateManager}
+   * @param properties properties to set
+   */
   public void simSetClusterProperties(Map<String, Object> properties) throws Exception
{
     lock.lock();
     try {
@@ -756,6 +859,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Set a cluster property. This also updates the clusterprops.json data in
+   * {@link DistribStateManager}
+   * @param key property name
+   * @param value property value
+   */
   public void simSetClusterProperty(String key, Object value) throws Exception {
     lock.lock();
     try {
@@ -770,6 +879,11 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Set collection properties.
+   * @param coll collection name
+   * @param properties properties
+   */
   public void simSetCollectionProperties(String coll, Map<String, Object> properties)
throws Exception {
     if (properties == null) {
       collProperties.remove(coll);
@@ -787,6 +901,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Set collection property.
+   * @param coll collection name
+   * @param key property name
+   * @param value property value
+   */
   public void simSetCollectionProperty(String coll, String key, String value) throws Exception
{
     Map<String, Object> props = collProperties.computeIfAbsent(coll, c -> new HashMap<>());
     if (value == null) {
@@ -797,6 +917,12 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     saveClusterState();
   }
 
+  /**
+   * Set slice properties.
+   * @param coll collection name
+   * @param slice slice name
+   * @param properties slice properties
+   */
   public void simSetSliceProperties(String coll, String slice, Map<String, Object>
properties) throws Exception {
     Map<String, Object> sliceProps = sliceProperties.computeIfAbsent(coll, c ->
new HashMap<>()).computeIfAbsent(slice, s -> new HashMap<>());
     lock.lock();
@@ -811,11 +937,56 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
     }
   }
 
+  /**
+   * Set per-collection value (eg. a metric). This value will be applied to each replica.
+   * @param collection collection name
+   * @param key property name
+   * @param value property value
+   */
+  public void simSetCollectionValue(String collection, String key, Object value) throws Exception
{
+    simSetCollectionValue(collection, key, value, false);
+  }
+
+  /**
+   * Set per-collection value (eg. a metric). This value will be applied to each replica.
+   * @param collection collection name
+   * @param key property name
+   * @param value property value
+   * @param divide if the value is a {@link Number} and this is true, then the value will
be evenly
+   *               divided by the number of replicas.
+   */
   public void simSetCollectionValue(String collection, String key, Object value, boolean
divide) throws Exception {
+    simSetShardValue(collection, null, key, value, divide);
+  }
+
+  /**
+   * Set per-collection value (eg. a metric). This value will be applied to each replica
in a selected shard.
+   * @param collection collection name
+   * @param shard shard name. If null then all shards will be affected.
+   * @param key property name
+   * @param value property value
+   */
+  public void simSetShardValue(String collection, String shard, String key, Object value)
throws Exception {
+    simSetShardValue(collection, shard, key, value, false);
+  }
+
+  /**
+   * Set per-collection value (eg. a metric). This value will be applied to each replica
in a selected shard.
+   * @param collection collection name
+   * @param shard shard name. If null then all shards will be affected.
+   * @param key property name
+   * @param value property value
+   * @param divide if the value is a {@link Number} and this is true, then the value will
be evenly
+   *               divided by the number of replicas.
+   */
+  public void simSetShardValue(String collection, String shard, String key, Object value,
boolean divide) throws Exception {
     List<ReplicaInfo> infos = new ArrayList<>();
     nodeReplicaMap.forEach((n, replicas) -> {
       replicas.forEach(r -> {
         if (r.getCollection().equals(collection)) {
+          if (shard != null && !shard.equals(r.getShard())) {
+            return;
+          }
           infos.add(r);
         }
       });
@@ -827,23 +998,29 @@ public class SimClusterStateProvider implements ClusterStateProvider
{
       value = ((Number)value).doubleValue() / infos.size();
     }
     for (ReplicaInfo r : infos) {
-      if (value == null) {
-        r.getVariables().remove(key);
-      } else {
-        r.getVariables().put(key, value);
+      synchronized (r) {
+        if (value == null) {
+          r.getVariables().remove(key);
+        } else {
+          r.getVariables().put(key, value);
+        }
       }
     }
   }
 
-  public void simSetCollectionValue(String collection, String key, Object value) throws Exception
{
-    simSetCollectionValue(collection, key, value, false);
-  }
-
-
+  /**
+   * Return all replica infos for a node.
+   * @param node node id
+   * @return list of replicas on that node
+   */
   public List<ReplicaInfo> simGetReplicaInfos(String node) {
     return nodeReplicaMap.get(node);
   }
 
+  /**
+   * List collections.
+   * @return list of existing collections.
+   */
   public List<String> simListCollections() {
     final Set<String> collections = new HashSet<>();
     lock.lock();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
index ea00e6c..e9616f0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java
@@ -46,7 +46,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Simulated {@link DistributedQueueFactory} that keeps all data in memory.
+ * Simulated {@link DistributedQueueFactory} that keeps all data in memory. Unlike
+ * the {@link GenericDistributedQueueFactory} this queue implementation data is not
+ * exposed anywhere.
  */
 public class SimDistributedQueueFactory implements DistributedQueueFactory {
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
index 577c976..65fb5b9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java
@@ -61,6 +61,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
   }
 
   // -------- simulator setup methods ------------
+
+  /**
+   * Get a node value
+   * @param node node id
+   * @param key property name
+   * @return property value or null if property or node doesn't exist.
+   */
   public Object simGetNodeValue(String node, String key) {
     Map<String, Object> values = nodeValues.get(node);
     if (values == null) {
@@ -69,6 +76,12 @@ public class SimNodeStateProvider implements NodeStateProvider {
     return values.get(key);
   }
 
+  /**
+   * Set node values.
+   * NOTE: if values contain 'nodeRole' key then /roles.json is updated.
+   * @param node node id
+   * @param values values.
+   */
   public void simSetNodeValues(String node, Map<String, Object> values) {
     Map<String, Object> existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
     existing.clear();
@@ -80,6 +93,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
     }
   }
 
+  /**
+   * Set a node value, replacing any previous value.
+   * NOTE: if key is 'nodeRole' then /roles.json is updated.
+   * @param node node id
+   * @param key property name
+   * @param value property value
+   */
   public void simSetNodeValue(String node, String key, Object value) {
     Map<String, Object> existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
     if (value == null) {
@@ -92,6 +112,13 @@ public class SimNodeStateProvider implements NodeStateProvider {
     }
   }
 
+  /**
+   * Add a node value, creating a list of values if necessary.
+   * NOTE: if key is 'nodeRole' then /roles.json is updated.
+   * @param node node id
+   * @param key property name
+   * @param value property value.
+   */
   public void simAddNodeValue(String node, String key, Object value) {
     Map<String, Object> values = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>());
     Object existing = values.get(key);
@@ -110,6 +137,11 @@ public class SimNodeStateProvider implements NodeStateProvider {
     }
   }
 
+  /**
+   * Remove node values. If values contained a 'nodeRole' key then
+   * /roles.json is updated.
+   * @param node node id
+   */
   public void simRemoveNodeValues(String node) {
     Map<String, Object> values = nodeValues.remove(node);
     if (values != null && values.containsKey("nodeRole")) {
@@ -117,6 +149,9 @@ public class SimNodeStateProvider implements NodeStateProvider {
     }
   }
 
+  /**
+   * Get all node values.
+   */
   public Map<String, Map<String, Object>> simGetAllNodeValues() {
     return nodeValues;
   }
@@ -136,6 +171,14 @@ public class SimNodeStateProvider implements NodeStateProvider {
     }
   }
 
+  /**
+   * Simulate getting replica metrics values. This uses per-replica properties set in
+   * {@link SimClusterStateProvider#simSetCollectionValue(String, String, Object, boolean)}
and
+   * similar methods.
+   * @param node node id
+   * @param tags metrics names
+   * @return map of metrics names / values
+   */
   public Map<String, Object> getReplicaMetricsValues(String node, Collection<String>
tags) {
     List<ReplicaInfo> replicas = clusterStateProvider.simGetReplicaInfos(node);
     if (replicas == null || replicas.isEmpty()) {
@@ -182,7 +225,7 @@ public class SimNodeStateProvider implements NodeStateProvider {
 
   @Override
   public Map<String, Object> getNodeValues(String node, Collection<String> tags)
{
-    LOG.debug("-- requested values for " + node + ": " + tags);
+    LOG.trace("-- requested values for " + node + ": " + tags);
     if (!liveNodes.contains(node)) {
       nodeValues.remove(node);
       return Collections.emptyMap();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d2524e9/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
index 726f839..92e3a95 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestLargeCluster.java
@@ -21,9 +21,12 @@ import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -65,6 +68,7 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
 
   static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
   static AtomicInteger triggerFiredCount = new AtomicInteger();
+  static CountDownLatch triggerFiredLatch;
   static int waitForSeconds;
 
   @BeforeClass
@@ -82,6 +86,7 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
 
     waitForSeconds = 1 + random().nextInt(3);
     triggerFiredCount.set(0);
+    triggerFiredLatch = new CountDownLatch(1);
     listenerEvents.clear();
     // clear any events or markers
     removeChildren(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
@@ -113,12 +118,12 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
     @Override
     public void process(TriggerEvent event, ActionContext context) throws Exception {
       triggerFiredCount.incrementAndGet();
+      triggerFiredLatch.countDown();
     }
   }
 
   @Test
   public void testBasic() throws Exception {
-    assertEquals(NUM_NODES, cluster.getClusterStateProvider().getLiveNodes().size());
     SolrClient solrClient = cluster.simGetSolrClient();
     String setTriggerCommand = "{" +
         "'set-trigger' : {" +
@@ -179,11 +184,64 @@ public class TestLargeCluster extends SimSolrCloudTestCase {
     }
 
     log.info("Ready after " + waitForState(collectionName, 90 * KILL_NODES, TimeUnit.SECONDS,
clusterShape(2, 15)) + "ms");
-    log.info(cluster.simGetSystemCollection().toString());
+
   }
 
   @Test
   public void testSearchRate() throws Exception {
+    SolrClient solrClient = cluster.simGetSolrClient();
+    String setTriggerCommand = "{" +
+        "'set-trigger' : {" +
+        "'name' : 'search_rate_trigger'," +
+        "'event' : 'searchRate'," +
+        "'waitFor' : '" + waitForSeconds + "s'," +
+        "'rate' : 1.0," +
+        "'enabled' : true," +
+        "'actions' : [" +
+        "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
+        "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
+        "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" +
+        "]" +
+        "}}";
+    SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+    NamedList<Object> response = solrClient.request(req);
+    assertEquals(response.get("result").toString(), "success");
+    String setListenerCommand1 = "{" +
+        "'set-listener' : " +
+        "{" +
+        "'name' : 'srt'," +
+        "'trigger' : 'search_rate_trigger'," +
+        "'stage' : ['FAILED','SUCCEEDED']," +
+        "'class' : '" + TestTriggerListener.class.getName() + "'" +
+        "}" +
+        "}";
+    req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand1);
+    response = solrClient.request(req);
+    assertEquals(response.get("result").toString(), "success");
+
+    String collectionName = "testSearchRate";
+    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+        "conf", 2, 10);
+    create.process(solrClient);
 
+    log.info("Ready after " + waitForState(collectionName, 300, TimeUnit.SECONDS, clusterShape(2,
10)) + " ms");
+
+    // collect the node names
+    Set<String> nodes = new HashSet<>();
+    cluster.getSimClusterStateProvider().getClusterState().getCollection(collectionName)
+        .getReplicas()
+        .forEach(r -> nodes.add(r.getNodeName()));
+
+    String metricName = "QUERY./select.requestTimes:1minRate";
+    // simulate search traffic
+    cluster.getSimClusterStateProvider().simSetShardValue(collectionName, "shard1", metricName,
40, true);
+
+    Thread.sleep(1000000000);
+//    boolean await = triggerFiredLatch.await(20000 / SPEED, TimeUnit.MILLISECONDS);
+//    assertTrue("The trigger did not fire at all", await);
+    // wait for listener to capture the SUCCEEDED stage
+    cluster.getTimeSource().sleep(2000);
+    assertEquals(listenerEvents.toString(), 1, listenerEvents.get("srt").size());
+    CapturedEvent ev = listenerEvents.get("srt").get(0);
   }
 }


Mime
View raw message