lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject lucene-solr:master: SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method.
Date Wed, 31 Oct 2018 17:48:18 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/master 914c43198 -> 3ed64375e


SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3ed64375
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3ed64375
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3ed64375

Branch: refs/heads/master
Commit: 3ed64375efc1ce19152eac9f75fbdf9e56dcf1b6
Parents: 914c431
Author: Andrzej Bialecki <ab@apache.org>
Authored: Wed Oct 31 18:47:34 2018 +0100
Committer: Andrzej Bialecki <ab@apache.org>
Committed: Wed Oct 31 18:47:34 2018 +0100

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  2 +
 .../cloud/autoscaling/IndexSizeTrigger.java     | 25 +++++-
 .../solr/cloud/autoscaling/TriggerEvent.java    |  2 +
 .../solr/cloud/overseer/ReplicaMutator.java     |  4 +-
 .../cloud/autoscaling/IndexSizeTriggerTest.java | 81 ++++++++++++++++++++
 .../src/solrcloud-autoscaling-triggers.adoc     |  7 ++
 .../cloud/autoscaling/SplitShardSuggester.java  | 10 ++-
 .../solrj/cloud/autoscaling/Suggester.java      | 15 +++-
 8 files changed, 138 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f0b8e92..a6519e0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -172,6 +172,8 @@ New Features
 
 * SOLR-12862: Add log10 Stream Evaluator and allow the pow Stream Evaluator to accept a vector
of exponents (Joel Bernstein)
 
+* SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method. (ab)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 25083ae..31322c0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -41,11 +41,13 @@ import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
 import org.apache.solr.common.util.Pair;
 import org.apache.solr.common.util.StrUtils;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.update.SolrIndexSplitter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,6 +65,7 @@ public class IndexSizeTrigger extends TriggerBase {
   public static final String BELOW_OP_PROP = "belowOp";
   public static final String COLLECTIONS_PROP = "collections";
   public static final String MAX_OPS_PROP = "maxOps";
+  public static final String SPLIT_METHOD_PROP = CommonAdminParams.SPLIT_METHOD;
 
   public static final String BYTES_SIZE_PROP = "__bytes__";
   public static final String DOCS_SIZE_PROP = "__docs__";
@@ -76,6 +79,7 @@ public class IndexSizeTrigger extends TriggerBase {
 
   private long aboveBytes, aboveDocs, belowBytes, belowDocs;
   private int maxOps;
+  private SolrIndexSplitter.SplitMethod splitMethod;
   private CollectionParams.CollectionAction aboveOp, belowOp;
   private final Set<String> collections = new HashSet<>();
   private final Map<String, Long> lastAboveEventMap = new ConcurrentHashMap<>();
@@ -85,7 +89,7 @@ public class IndexSizeTrigger extends TriggerBase {
     super(TriggerEventType.INDEXSIZE, name);
     TriggerUtils.validProperties(validProperties,
         ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
-        COLLECTIONS_PROP, MAX_OPS_PROP);
+        COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_METHOD_PROP);
   }
 
   @Override
@@ -165,6 +169,12 @@ public class IndexSizeTrigger extends TriggerBase {
     } catch (Exception e) {
       throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" +
maxOpsStr + "': " + e.getMessage());
     }
+    String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower());
+    splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr);
+    if (splitMethod == null) {
+      throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "Unknown value '"
+ CommonAdminParams.SPLIT_METHOD +
+          ": " + methodStr);
+    }
   }
 
   @Override
@@ -234,7 +244,7 @@ public class IndexSizeTrigger extends TriggerBase {
           DocCollection docCollection = clusterState.getCollection(coll);
 
           shards.forEach((sh, replicas) -> {
-            // check only the leader of a replica in active shard
+            // check only the leader replica in an active shard
             Slice s = docCollection.getSlice(sh);
             if (s.getState() != Slice.State.ACTIVE) {
               return;
@@ -244,6 +254,10 @@ public class IndexSizeTrigger extends TriggerBase {
             if (r == null) {
               return;
             }
+            // not on this node
+            if (!r.getNodeName().equals(node)) {
+              return;
+            }
             // find ReplicaInfo
             ReplicaInfo info = null;
             for (ReplicaInfo ri : replicas) {
@@ -279,14 +293,14 @@ public class IndexSizeTrigger extends TriggerBase {
           } else {
             // verify that it's a Number
             if (!(size instanceof Number)) {
-              log.warn("invalid size value - not a number: '" + size + "' is " + size.getClass().getName());
+              log.warn("invalid size value for tag " + tag + " - not a number: '" + size
+ "' is " + size.getClass().getName());
               return;
             }
 
             ReplicaInfo currentInfo = currentSizes.computeIfAbsent(info.getCore(), k ->
(ReplicaInfo)info.clone());
             if (tag.contains("INDEX")) {
               currentInfo.getVariables().put(BYTES_SIZE_PROP, ((Number) size).longValue());
-            } else {
+            } else if (tag.contains("SEARCHER")) {
               currentInfo.getVariables().put(DOCS_SIZE_PROP, ((Number) size).longValue());
             }
           }
@@ -383,6 +397,9 @@ public class IndexSizeTrigger extends TriggerBase {
         }
         TriggerEvent.Op op = new TriggerEvent.Op(aboveOp);
         op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard()));
+        Map<String, Object> params = new HashMap<>();
+        params.put(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
+        op.addHint(Suggester.Hint.PARAMS, params);
         ops.add(op);
         Long time = lastAboveEventMap.get(r.getCore());
         if (time != null && eventTime.get() > time) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
index 8e3a348..c61556c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
@@ -64,6 +64,8 @@ public class TriggerEvent implements MapWriter {
       if (hint.multiValued) {
         Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
         ((Set) hints.computeIfAbsent(hint, h -> new LinkedHashSet<>())).addAll(values);
+      } else if (value instanceof Map) {
+        hints.put(hint, value);
       } else {
         hints.put(hint, value == null ? null : String.valueOf(value));
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
index 68a42b9..82a0610 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
@@ -394,7 +394,7 @@ public class ReplicaMutator {
           }
         }
         if (allActive) {
-          log.info("Shard: {} - all replicas are active. Finding status of fellow sub-shards",
sliceName);
+          log.info("Shard: {} - all {} replicas are active. Finding status of fellow sub-shards",
sliceName, slice.getReplicasMap().size());
           // find out about other sub shards
           Map<String, Slice> allSlicesCopy = new HashMap<>(collection.getSlicesMap());
           List<Slice> subShardSlices = new ArrayList<>();
@@ -413,7 +413,7 @@ public class ReplicaMutator {
                     break outer;
                   }
                 }
-                log.info("Shard: {} - Fellow sub-shard: {} has all replicas active", sliceName,
otherSlice.getName());
+                log.info("Shard: {} - Fellow sub-shard: {} has all {} replicas active", sliceName,
otherSlice.getName(), otherSlice.getReplicasMap().size());
                 subShardSlices.add(otherSlice);
               }
             }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 996532a..ec2315d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -46,6 +46,7 @@ import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.NamedList;
@@ -207,6 +208,9 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
         } else {
           fail("unexpected shard name " + p.second());
         }
+        Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
+        assertNotNull("params are null: " + op, params);
+        assertEquals("splitMethod: " + op, "rewrite", params.get(CommonAdminParams.SPLIT_METHOD));
       }
       assertTrue("shard1 should be split", shard1);
       assertTrue("shard2 should be split", shard2);
@@ -822,6 +826,83 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
     assertEquals("number of ops: " + ops, 3, ops.size());
   }
 
+  @Test
+  public void testSplitMethodConfig() throws Exception {
+    String collectionName = "testSplitMethod_collection";
+    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+        "conf", 2, 2).setMaxShardsPerNode(2);
+    create.process(solrClient);
+    CloudTestUtils.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
+        CloudTestUtils.clusterShape(2, 2, false, true));
+
+    long waitForSeconds = 3 + random().nextInt(5);
+    Map<String, Object> props = createTriggerProps(waitForSeconds);
+    props.put(CommonAdminParams.SPLIT_METHOD, "link");
+    try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger6")) {
+      trigger.configure(loader, cloudManager, props);
+      trigger.init();
+      trigger.setProcessor(noFirstRunProcessor);
+      trigger.run();
+
+      for (int i = 0; i < 25; i++) {
+        SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
+        solrClient.add(collectionName, doc);
+      }
+      solrClient.commit(collectionName);
+
+      AtomicBoolean fired = new AtomicBoolean(false);
+      AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
+      trigger.setProcessor(event -> {
+        if (fired.compareAndSet(false, true)) {
+          eventRef.set(event);
+          long currentTimeNanos = timeSource.getTimeNs();
+          long eventTimeNanos = event.getEventTime();
+          long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS)
- WAIT_FOR_DELTA_NANOS;
+          if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+            fail("processor was fired before the configured waitFor period: currentTimeNanos="
+ currentTimeNanos + ", eventTimeNanos=" +  eventTimeNanos + ",waitForNanos=" + waitForNanos);
+          }
+        } else {
+          fail("IndexSizeTrigger was fired more than once!");
+        }
+        return true;
+      });
+      trigger.run();
+      TriggerEvent ev = eventRef.get();
+      // waitFor delay - should not produce any event yet
+      assertNull("waitFor not elapsed but produced an event", ev);
+      timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+      trigger.run();
+      ev = eventRef.get();
+      assertNotNull("should have fired an event", ev);
+      List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) ev.getProperty(TriggerEvent.REQUESTED_OPS);
+      assertNotNull("should contain requestedOps", ops);
+      assertEquals("number of ops: " + ops, 2, ops.size());
+      boolean shard1 = false;
+      boolean shard2 = false;
+      for (TriggerEvent.Op op : ops) {
+        assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
+        Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
+        assertNotNull("hints", hints);
+        assertEquals("hints", 1, hints.size());
+        Pair<String, String> p = hints.iterator().next();
+        assertEquals(collectionName, p.first());
+        if (p.second().equals("shard1")) {
+          shard1 = true;
+        } else if (p.second().equals("shard2")) {
+          shard2 = true;
+        } else {
+          fail("unexpected shard name " + p.second());
+        }
+        Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
+        assertNotNull("params are null: " + op, params);
+        assertEquals("splitMethod: " + op, "link", params.get(CommonAdminParams.SPLIT_METHOD));
+      }
+      assertTrue("shard1 should be split", shard1);
+      assertTrue("shard2 should be split", shard2);
+    }
+
+  }
+
   private Map<String, Object> createTriggerProps(long waitForSeconds) {
     Map<String, Object> props = new HashMap<>();
     props.put("event", "indexSize");

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 97b9dd7..d091537 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -304,6 +304,13 @@ changes in a highly dynamic situation, which may lead to more serious
threshold
 but it also limits the maximum load on the cluster that the large number of requested
 operations may cause. The default value is 10.
 
+`splitMethod`::
+One of the supported methods for index splitting to use. Default value is `rewrite`, which
is
+slow and puts a high CPU load on the shard leader but results in optimized sub-shard indexes.
+The `link` method is much faster and puts very little load on the shard leader but results
in
+indexes that are initially as large as the parent shard's index, which slows down replication
and
+may lead to excessive initial disk space consumption on replicas.
+
 Events generated by this trigger contain additional details about the shards
 that exceeded thresholds and the types of violations (upper / lower bounds, bytes / docs
metrics).
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
index 2c1d7df..559ff8a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
@@ -17,11 +17,13 @@
 package org.apache.solr.client.solrj.cloud.autoscaling;
 
 import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
 import org.apache.solr.common.util.Pair;
 
 /**
@@ -44,6 +46,12 @@ class SplitShardSuggester extends Suggester {
       throw new RuntimeException("split-shard requires exactly one pair of 'collection' and
'shard'");
     }
     Pair<String, String> collShard = shards.iterator().next();
-    return CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+    Map<String, Object> params = (Map<String, Object>)hints.getOrDefault(Hint.PARAMS,
Collections.emptyMap());
+    CollectionAdminRequest.SplitShard req = CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+    String splitMethod = (String)params.get(CommonAdminParams.SPLIT_METHOD);
+    if (splitMethod != null) {
+      req.setSplitMethod(splitMethod);
+    }
+    return req;
   }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3ed64375/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index bba5906..39ad8bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -95,7 +95,15 @@ public abstract class Suggester implements MapWriter {
       Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
       ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).addAll(values);
     } else {
-      hints.put(hint, value == null ? null : String.valueOf(value));
+      if (value == null) {
+        hints.put(hint, null);
+      } else {
+        if ((value instanceof Map) || (value instanceof Number)) {
+          hints.put(hint, value);
+        } else {
+          hints.put(hint, String.valueOf(value));
+        }
+      }
     }
     return this;
   }
@@ -372,6 +380,11 @@ public abstract class Suggester implements MapWriter {
     NUMBER(true, o -> {
       if (!(o instanceof Number)) throw new RuntimeException("NUMBER hint must be a number");
     }),
+    PARAMS(false, o -> {
+      if (!(o instanceof Map)) {
+        throw new RuntimeException("PARAMS hint must be a Map<String, Object>");
+      }
+    }),
     REPLICA(true);
 
     public final boolean multiValued;


Mime
View raw message