lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject lucene-solr:jira/solr-11031: SOLR-11031: Implement LogPlanAction for autoscaling.
Date Mon, 17 Jul 2017 14:21:13 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-11031 [created] 99da42e2b


SOLR-11031: Implement LogPlanAction for autoscaling.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/99da42e2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/99da42e2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/99da42e2

Branch: refs/heads/jira/solr-11031
Commit: 99da42e2b784426d7c0cb3aaf8a716527f522af1
Parents: 2590a43
Author: Andrzej Bialecki <ab@apache.org>
Authored: Mon Jul 17 15:57:12 2017 +0200
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Mon Jul 17 15:57:12 2017 +0200

----------------------------------------------------------------------
 .../apache/solr/cloud/CreateCollectionCmd.java  |   7 +-
 .../solr/cloud/autoscaling/ActionContext.java   |  14 +-
 .../solr/cloud/autoscaling/LogPlanAction.java   |  91 +++++++++++-
 .../org/apache/solr/core/BlobRepository.java    |  12 +-
 .../org/apache/solr/core/MemClassLoader.java    |   3 +-
 .../src/java/org/apache/solr/core/SolrCore.java |   4 +-
 .../org/apache/solr/handler/BlobHandler.java    |   1 +
 .../solr/handler/admin/CollectionsHandler.java  |  10 +-
 .../org/apache/solr/servlet/HttpSolrCall.java   |   8 +-
 .../src/java/org/apache/solr/util/SolrCLI.java  |   3 +-
 .../src/resources/SystemCollectionSchema.xml    |  27 +++-
 .../cloud/autoscaling/LogPlanActionTest.java    | 146 +++++++++++++++++++
 .../solr/core/BlobRepositoryCloudTest.java      |   3 +-
 .../solrj/request/CollectionAdminRequest.java   |  20 ++-
 .../common/params/CollectionAdminParams.java    |   4 +-
 .../apache/solr/common/params/CommonParams.java |   3 +
 16 files changed, 324 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
index d859170..cc1c1a1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
@@ -30,7 +30,6 @@ import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.request.V2Request;
 import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
 import org.apache.solr.cloud.autoscaling.AutoScaling;
 import org.apache.solr.cloud.autoscaling.AutoScalingHandler;
@@ -48,18 +47,16 @@ import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.cloud.ZooKeeperException;
 import org.apache.solr.common.params.AutoScalingParams;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CoreAdminParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.CommandOperation;
-import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.handler.admin.ConfigSetsHandlerApi;
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.handler.component.ShardRequest;
-import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.util.TimeOut;
 import org.apache.zookeeper.CreateMode;
@@ -349,7 +346,7 @@ public class CreateCollectionCmd implements Cmd {
       try {
         configNames = ocmh.zkStateReader.getZkClient().getChildren(ZkConfigManager.CONFIGS_ZKNODE,
null, true);
         if (configNames.contains(ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME)) {
-          if (!".system".equals(coll)) {
+          if (!CollectionAdminParams.SYSTEM_COLL.equals(coll)) {
             copyDefaultConfigSetTo(configNames, coll);
           }
           return coll;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java
index ecc339c..178a972 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java
@@ -17,8 +17,10 @@
 
 package org.apache.solr.cloud.autoscaling;
 
+import java.io.IOException;
 import java.util.Map;
 
+import org.apache.solr.common.MapWriter;
 import org.apache.solr.core.CoreContainer;
 
 /**
@@ -26,7 +28,7 @@ import org.apache.solr.core.CoreContainer;
  * which the action is being executed as well as helper methods to pass computed information
along
  * to the next action
  */
-public class ActionContext {
+public class ActionContext implements MapWriter {
 
   private final CoreContainer coreContainer;
   private final AutoScaling.Trigger source;
@@ -53,4 +55,14 @@ public class ActionContext {
   public Object getProperty(String name)  {
     return properties != null ? properties.get(name) : null;
   }
+
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    ew.put("source", source.getName());
+    if (properties != null) {
+      for (Map.Entry<String, Object> entry : properties.entrySet()) {
+        ew.put("properties." + entry.getKey(), entry.getValue());
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/cloud/autoscaling/LogPlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/LogPlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/LogPlanAction.java
index 45107c1..3a043f8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/LogPlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/LogPlanAction.java
@@ -17,12 +17,101 @@
 
 package org.apache.solr.cloud.autoscaling;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.StringJoiner;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+
 /**
- * todo nocommit
+ * This action saves the computed action plan to a {@link CollectionAdminParams#SYSTEM_COLL}
collection.
  */
 public class LogPlanAction extends TriggerActionBase {
+  public static final String SOURCE_FIELD = "source_s";
+  public static final String SOURCE = LogPlanAction.class.getSimpleName();
+  public static final String DOC_TYPE = "autoscaling_action";
+
   @Override
   public void process(TriggerEvent event, ActionContext actionContext) {
+    CoreContainer container = actionContext.getCoreContainer();
+    try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
+        .withZkHost(container.getZkController().getZkServerAddress())
+        .withHttpClient(container.getUpdateShardHandler().getHttpClient())
+        .build()) {
+      SolrInputDocument doc = new SolrInputDocument();
+      doc.addField(CommonParams.TYPE, DOC_TYPE);
+      doc.addField(SOURCE_FIELD, SOURCE);
+      doc.addField("id", event.getId());
+      doc.addField("event.type_s", event.getEventType().toString());
+      doc.addField("event.source_s", event.getSource());
+      doc.addField("event.time_l", event.getEventTime());
+      doc.addField("timestamp", new Date());
+      addMap("event.property.", doc, event.getProperties());
+      addOperations(doc, (List<SolrRequest>)actionContext.getProperties().get("operations"));
+      // add JSON versions of event and context
+      String eventJson = Utils.toJSONString(event);
+      String contextJson = Utils.toJSONString(actionContext);
+      doc.addField("event_str", eventJson);
+      doc.addField("context_str", contextJson);
+      UpdateRequest req = new UpdateRequest();
+      req.add(doc);
+      cloudSolrClient.request(req, CollectionAdminParams.SYSTEM_COLL);
+    } catch (Exception e) {
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+          "Unexpected exception while processing event: " + event, e);
+    }
+  }
+
+  private void addMap(String prefix, SolrInputDocument doc, Map<String, Object> map)
{
+    map.forEach((k, v) -> {
+      if (v instanceof Collection) {
+        for (Object o : (Collection)v) {
+          doc.addField(prefix + k + "_ss", String.valueOf(o));
+        }
+      } else {
+        doc.addField(prefix + k + "_s", String.valueOf(v));
+      }
+    });
+  }
 
+  private void addOperations(SolrInputDocument doc, List<SolrRequest> operations) {
+    if (operations == null || operations.isEmpty()) {
+      return;
+    }
+    for (SolrRequest req : operations) {
+      SolrParams params = req.getParams();
+      if (params == null) {
+        continue;
+      }
+      // build a whitespace-separated param string
+      StringJoiner paramJoiner = new StringJoiner(" ");
+      paramJoiner.setEmptyValue("");
+      for (Iterator<String> it = params.getParameterNamesIterator(); it.hasNext();
) {
+        final String name = it.next();
+        final String [] values = params.getParams(name);
+        for (String value : values) {
+          paramJoiner.add(name + "=" + value);
+        }
+      }
+      String paramString = paramJoiner.toString();
+      if (!paramString.isEmpty()) {
+        doc.addField("operations.params_ts", paramString);
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/core/BlobRepository.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
index bbe7c62..e4d5e04 100644
--- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java
@@ -42,7 +42,7 @@ import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.util.SimplePostTool;
 import org.apache.zookeeper.server.ByteBufferInputStream;
 import org.slf4j.Logger;
@@ -153,7 +153,7 @@ public class BlobRepository {
    */
   ByteBuffer fetchBlob(String key) {
     Replica replica = getSystemCollReplica();
-    String url = replica.getStr(BASE_URL_PROP) + "/.system/blob/" + key + "?wt=filestream";
+    String url = replica.getStr(BASE_URL_PROP) + "/" + CollectionAdminParams.SYSTEM_COLL
+ "/blob/" + key + "?wt=filestream";
 
     HttpClient httpClient = coreContainer.getUpdateShardHandler().getHttpClient();
     HttpGet httpGet = new HttpGet(url);
@@ -180,10 +180,10 @@ public class BlobRepository {
   private Replica getSystemCollReplica() {
     ZkStateReader zkStateReader = this.coreContainer.getZkController().getZkStateReader();
     ClusterState cs = zkStateReader.getClusterState();
-    DocCollection coll = cs.getCollectionOrNull(CollectionsHandler.SYSTEM_COLL);
-    if (coll == null) throw new SolrException(SERVICE_UNAVAILABLE, ".system collection not
available");
+    DocCollection coll = cs.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL);
+    if (coll == null) throw new SolrException(SERVICE_UNAVAILABLE, CollectionAdminParams.SYSTEM_COLL
+ " collection not available");
     ArrayList<Slice> slices = new ArrayList<>(coll.getActiveSlices());
-    if (slices.isEmpty()) throw new SolrException(SERVICE_UNAVAILABLE, "No active slices
for .system collection");
+    if (slices.isEmpty()) throw new SolrException(SERVICE_UNAVAILABLE, "No active slices
for " + CollectionAdminParams.SYSTEM_COLL + " collection");
     Collections.shuffle(slices, RANDOM); //do load balancing
 
     Replica replica = null;
@@ -202,7 +202,7 @@ public class BlobRepository {
       }
     }
     if (replica == null) {
-      throw new SolrException(SERVICE_UNAVAILABLE, ".no active replica available for .system
collection");
+      throw new SolrException(SERVICE_UNAVAILABLE, "No active replica available for " + CollectionAdminParams.SYSTEM_COLL
+ " collection");
     }
     return replica;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
index 4df0a8b..e4f561f 100644
--- a/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/MemClassLoader.java
@@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -92,7 +93,7 @@ public class MemClassLoader extends ClassLoader implements AutoCloseable,
Resour
     ProtectionDomain defaultDomain = null;
     //using the default protection domain, with no permissions
     try {
-      defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/.system/blob/"
+ jarName.get()), (Certificate[]) null),
+      defaultDomain = new ProtectionDomain(new CodeSource(new URL("http://localhost/" + CollectionAdminParams.SYSTEM_COLL
+ "/blob/" + jarName.get()), (Certificate[]) null),
           null);
     } catch (MalformedURLException mue) {
       throw new ClassNotFoundException("Unexpected exception ", mue);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 5319881..7ff1a52 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -82,6 +82,7 @@ import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.CommonParams.EchoParamStyle;
 import org.apache.solr.common.params.SolrParams;
@@ -3035,7 +3036,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer,
Closeab
    * processing. The Decoder will only run on the first invocations, subsequent invocations
will return the 
    * cached object. 
    * 
-   * @param key A key in the format of name/version for a blob stored in the .system blob
store via the Blob Store API
+   * @param key A key in the format of name/version for a blob stored in the
+   *            {@link CollectionAdminParams#SYSTEM_COLL} blob store via the Blob Store API
    * @param decoder a decoder with which to convert the blob into a Java Object representation
(first time only)
    * @return a reference to the blob that has already cached the decoded version.
    */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index 725a0ea..e3097fc 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -137,6 +137,7 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial
         String id = blobName + "/" + version;
         Map<String, Object> doc = makeMap(
             ID, id,
+            CommonParams.TYPE, "blob",
             "md5", md5,
             "blobName", blobName,
             VERSION, version,

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index a4155ea..5175fd9 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -348,20 +348,18 @@ public class CollectionsHandler extends RequestHandlerBase implements
Permission
     return Category.ADMIN;
   }
 
-  public static final String SYSTEM_COLL = ".system";
-
   private static void createSysConfigSet(CoreContainer coreContainer) throws KeeperException,
InterruptedException {
     SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient();
     ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zk.getZkClientTimeout());
     cmdExecutor.ensureExists(ZkStateReader.CONFIGS_ZKNODE, zk);
-    cmdExecutor.ensureExists(ZkStateReader.CONFIGS_ZKNODE + "/" + SYSTEM_COLL, zk);
+    cmdExecutor.ensureExists(ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL,
zk);
 
     try {
-      String path = ZkStateReader.CONFIGS_ZKNODE + "/" + SYSTEM_COLL + "/schema.xml";
+      String path = ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL
+ "/schema.xml";
       byte[] data = IOUtils.toByteArray(CollectionsHandler.class.getResourceAsStream("/SystemCollectionSchema.xml"));
       assert data != null && data.length > 0;
       cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk);
-      path = ZkStateReader.CONFIGS_ZKNODE + "/" + SYSTEM_COLL + "/solrconfig.xml";
+      path = ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL + "/solrconfig.xml";
       data = IOUtils.toByteArray(CollectionsHandler.class.getResourceAsStream("/SystemCollectionSolrConfig.xml"));
       assert data != null && data.length > 0;
       cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk);
@@ -418,7 +416,7 @@ public class CollectionsHandler extends RequestHandlerBase implements
Permission
       if (StringUtils.isNotEmpty(shardsParam)) {
         verifyShardsParam(shardsParam);
       }
-      if (SYSTEM_COLL.equals(collectionName)) {
+      if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) {
         //We must always create a .system collection with only a single shard
         props.put(NUM_SLICES, 1);
         props.remove(SHARDS_PROP);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index a548c05..e5ec720 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -119,7 +119,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.DE
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
 import static org.apache.solr.common.params.CommonParams.NAME;
 import static org.apache.solr.common.params.CoreAdminParams.ACTION;
-import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL;
+import static org.apache.solr.common.params.CollectionAdminParams.SYSTEM_COLL;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD;
 import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH;
@@ -347,7 +347,7 @@ public class HttpSolrCall {
         SYSTEM_COLL.equals(corename) &&
         "POST".equals(req.getMethod()) &&
         !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
-      log.info("Going to auto-create .system collection");
+      log.info("Going to auto-create " + SYSTEM_COLL + " collection");
       SolrQueryResponse rsp = new SolrQueryResponse();
       String repFactor = String.valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));
       cores.getCollectionsHandler().handleRequestBody(new LocalSolrQueryRequest(null,
@@ -356,7 +356,7 @@ public class HttpSolrCall {
               .add( NAME, SYSTEM_COLL)
               .add(REPLICATION_FACTOR, repFactor)), rsp);
       if (rsp.getValues().get("success") == null) {
-        throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create .system collection:
"+ Utils.toJSONString(rsp.getValues()));
+        throw new SolrException(ErrorCode.SERVER_ERROR, "Could not auto-create " + SYSTEM_COLL
+ " collection: "+ Utils.toJSONString(rsp.getValues()));
       }
       TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS);
       for (; ; ) {
@@ -364,7 +364,7 @@ public class HttpSolrCall {
           break;
         } else {
           if (timeOut.hasTimedOut()) {
-            throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find .system collection
even after 3 seconds");
+            throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find " + SYSTEM_COLL
+ " collection even after 3 seconds");
           }
           Thread.sleep(50);
         }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/java/org/apache/solr/util/SolrCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index 657b402..a4b4fdd 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -112,6 +112,7 @@ import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkConfigManager;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.ContentStreamBase;
@@ -1512,7 +1513,7 @@ public class SolrCLI {
       boolean configExistsInZk = confname != null && !"".equals(confname.trim())
&&
           cloudSolrClient.getZkStateReader().getZkClient().exists("/configs/" + confname,
true);
 
-      if (".system".equals(collectionName)) {
+      if (CollectionAdminParams.SYSTEM_COLL.equals(collectionName)) {
         //do nothing
       } else if (configExistsInZk) {
         echo("Re-using existing configuration directory "+confname);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/resources/SystemCollectionSchema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/resources/SystemCollectionSchema.xml b/solr/core/src/resources/SystemCollectionSchema.xml
index d491480..2077b36 100644
--- a/solr/core/src/resources/SystemCollectionSchema.xml
+++ b/solr/core/src/resources/SystemCollectionSchema.xml
@@ -2,16 +2,37 @@
 <schema name="_system collection or core" version="1.1">
   <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
   <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/>
   <fieldType name="bytes" class="solr.BinaryField"/>
   <fieldType name="date" class="solr.TrieDateField"/>
+  <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+  </fieldType>
+  <uniqueKey>id</uniqueKey>
   <field name="id"   type="string"   indexed="true"  stored="true"  multiValued="false"
required="true"/>
-  <field name="md5"   type="string"   indexed="true"  stored="true"  multiValued="false"
required="true"/>
+  <field name="_version_" type="long"     indexed="true"  stored="true"/>
+  <field name="type"   type="string"   indexed="true"  stored="true"  multiValued="false"
required="true"/>
+  <!-- blob repository fields -->
+  <field name="md5"   type="string"   indexed="true"  stored="true"  multiValued="false"/>
   <field name="blob"      type="bytes"   indexed="false" stored="true"  multiValued="false"
/>
   <field name="size"      type="long"   indexed="true" stored="true"  multiValued="false"
/>
   <field name="version"   type="long"     indexed="true"  stored="true"  multiValued="false"
/>
   <field name="timestamp"   type="date"   indexed="true"  stored="true"  multiValued="false"
/>
   <field name="blobName"      type="string"   indexed="true"  stored="true"  multiValued="false"
/>
-  <field name="_version_" type="long"     indexed="true"  stored="true"/>
+
+  <!-- general purpose fields -->
   <dynamicField name="*_s"  type="string"  indexed="true"  stored="true" />
-  <uniqueKey>id</uniqueKey>
+  <dynamicField name="*_ss"  type="string"  indexed="true"  multiValued="true"  stored="true"
/>
+  <dynamicField name="*_str"  type="string"  indexed="false" stored="true" />
+  <dynamicField name="*_strs"  type="string"  indexed="false"  multiValued="true" stored="true"
/>
+  <dynamicField name="*_bin"  type="bytes"  indexed="false"  multiValued="false" stored="true"
/>
+  <dynamicField name="*_t"  type="text_ws"  indexed="true"  stored="true" />
+  <dynamicField name="*_ts"  type="text_ws"  indexed="true"  multiValued="true" stored="true"
/>
+  <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+  <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
+  <dynamicField name="*_ls" type="long" multiValued="true" indexed="true" stored="true"/>
+  <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
+  <dynamicField name="*_ds" type="double" multiValued="true" indexed="true" stored="true"/>
 </schema>

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/test/org/apache/solr/cloud/autoscaling/LogPlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/LogPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/LogPlanActionTest.java
new file mode 100644
index 0000000..0e3d367
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/LogPlanActionTest.java
@@ -0,0 +1,146 @@
+package org.apache.solr.cloud.autoscaling;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+
+/**
+ * Test for {@link LogPlanAction}
+ */
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+public class LogPlanActionTest extends SolrCloudTestCase {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private static final AtomicBoolean fired = new AtomicBoolean(false);
+  private static final int NODE_COUNT = 3;
+  private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
+  private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
+  private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
+
+  public static class AssertingTriggerAction implements TriggerAction {
+
+    @Override
+    public String getName() {
+      return null;
+    }
+
+    @Override
+    public void process(TriggerEvent event, ActionContext context) {
+      if (fired.compareAndSet(false, true)) {
+        eventRef.set(event);
+        actionContextPropsRef.set(context.getProperties());
+        triggerFiredLatch.countDown();
+      }
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+
+    @Override
+    public void init(Map<String, String> args) {
+
+    }
+  }
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(NODE_COUNT)
+        .addConfig("conf", configset("cloud-minimal"))
+        .configure();
+    CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 3)
+        .process(cluster.getSolrClient());
+  }
+
+  @Test
+  public void test() throws Exception {
+    CloudSolrClient solrClient = cluster.getSolrClient();
+    String setTriggerCommand = "{" +
+        "'set-trigger' : {" +
+        "'name' : 'node_lost_trigger'," +
+        "'event' : 'nodeLost'," +
+        "'waitFor' : '1s'," +
+        "'enabled' : true," +
+        "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
+        "{'name':'log_plan','class':'solr.LogPlanAction'}," +
+        "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
+        "}}";
+    SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+    NamedList<Object> response = solrClient.request(req);
+    assertEquals(response.get("result").toString(), "success");
+
+    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("test",
+        "conf",3, 2);
+    create.setMaxShardsPerNode(3);
+    create.process(solrClient);
+
+    waitForState("Timed out waiting for replicas of new collection to be active",
+        "test", clusterShape(3, 2));
+    // stop non-overseer node
+    NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+    String overseerLeader = (String) overSeerStatus.get("leader");
+    int nonOverseerLeaderIndex = 0;
+    for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+      JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+      if (!jetty.getNodeName().equals(overseerLeader)) {
+        nonOverseerLeaderIndex = i;
+      }
+    }
+    cluster.stopJettySolrRunner(nonOverseerLeaderIndex);
+    cluster.waitForAllNodes(30);
+    assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS));
+    assertTrue(fired.get());
+    Map context = actionContextPropsRef.get();
+    assertNotNull(context);
+    // make sure the event doc is committed
+    cluster.getSolrClient().commit(CollectionAdminParams.SYSTEM_COLL);
+    ModifiableSolrParams query = new ModifiableSolrParams();
+    query.add(CommonParams.Q, "type:" + LogPlanAction.DOC_TYPE);
+    QueryResponse resp = cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL,
query);
+    SolrDocumentList docs = resp.getResults();
+    assertNotNull(docs);
+    assertEquals("wrong number of events added to .system", 1, docs.size());
+    SolrDocument doc = docs.get(0);
+    assertEquals(LogPlanAction.class.getSimpleName(), doc.getFieldValue(LogPlanAction.SOURCE_FIELD));
+    assertEquals(LogPlanAction.DOC_TYPE, doc.getFieldValue(CommonParams.TYPE));
+    assertEquals("node_lost_trigger", doc.getFieldValue("event.source_s"));
+    assertNotNull(doc.getFieldValue("event.time_l"));
+    assertNotNull(doc.getFieldValue("timestamp"));
+    assertNotNull(doc.getFieldValue("event.property.nodeName_s"));
+    assertNotNull(doc.getFieldValue("event.property._enqueue_time__s"));
+    assertNotNull(doc.getFieldValue("event_str"));
+    assertNotNull(doc.getFieldValue("context_str"));
+    assertEquals("NODELOST", doc.getFieldValue("event.type_s"));
+    Collection<Object> vals = doc.getFieldValues("operations.params_ts");
+    assertEquals(3, vals.size());
+    for (Object val : vals) {
+      assertTrue(val.toString(), String.valueOf(val).contains("action=MOVEREPLICA"));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java
index ad3c8db..efa8e11 100644
--- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java
+++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java
@@ -31,6 +31,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.handler.TestBlobHandler;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -46,7 +47,7 @@ public class BlobRepositoryCloudTest extends SolrCloudTestCase {
         .configure();
 //    Thread.sleep(2000);
     HashMap<String, String> params = new HashMap<>();
-    CollectionAdminRequest.createCollection(".system", null, 1, 1)
+    CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1)
         .process(cluster.getSolrClient());
     // test component will fail if it cant' find a blob with this data by this name
     TestBlobHandler.postData(cluster.getSolrClient(), findLiveNodeURI(), "testResource",
ByteBuffer.wrap("foo,bar\nbaz,bam".getBytes(StandardCharsets.UTF_8)));

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 88100f3..de28d71 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -18,6 +18,7 @@ package org.apache.solr.client.solrj.request;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
@@ -32,6 +33,7 @@ import org.apache.solr.client.solrj.V2RequestSupport;
 import org.apache.solr.client.solrj.response.CollectionAdminResponse;
 import org.apache.solr.client.solrj.response.RequestStatusState;
 import org.apache.solr.client.solrj.util.SolrIdentifierValidator;
+import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.ImplicitDocRouter;
 import org.apache.solr.common.cloud.Replica;
@@ -56,7 +58,7 @@ import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SE
  *
  * @since solr 4.5
  */
-public abstract class CollectionAdminRequest<T extends CollectionAdminResponse> extends
SolrRequest<T> implements V2RequestSupport {
+public abstract class CollectionAdminRequest<T extends CollectionAdminResponse> extends
SolrRequest<T> implements V2RequestSupport, MapWriter {
 
   protected final CollectionAction action;
 
@@ -96,6 +98,22 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
     }
   }
 
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    ew.put("class", this.getClass().getName());
+    ew.put("method", getMethod().toString());
+    SolrParams params = getParams();
+    if (params != null) {
+      for (Iterator<String> it = params.getParameterNamesIterator(); it.hasNext();
) {
+        final String name = it.next();
+        final String [] values = params.getParams(name);
+        for (String value : values) {
+          ew.put("params." + name, value);
+        }
+      }
+    }
+  }
+
   /**
    * Base class for asynchronous collection admin requests
    */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
index ffc373b..b5f23e2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
@@ -28,6 +28,9 @@ public interface CollectionAdminParams {
 
   String COUNT_PROP = "count";
 
+  /** Predefined system collection name. */
+  String SYSTEM_COLL = ".system";
+
   /**
    * A parameter to specify list of Solr nodes to be used (e.g. for collection creation or
restore operation).
    */
@@ -59,5 +62,4 @@ public interface CollectionAdminParams {
    */
   public static final Collection<String> INDEX_BACKUP_STRATEGIES =
       Arrays.asList(COPY_FILES_STRATEGY, NO_INDEX_BACKUP_STRATEGY);
-
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/99da42e2/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index 3b24f22..d260281 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -233,6 +233,9 @@ public interface CommonParams {
   String TRUE = Boolean.TRUE.toString();
   String FALSE = Boolean.FALSE.toString();
 
+  /** document type in {@link CollectionAdminParams#SYSTEM_COLL} collection. **/
+  String TYPE = "type";
+
   /** Used as a local parameter on queries.  cache=false means don't check any query or filter
caches.
    * cache=true is the default.
    */


Mime
View raw message