lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g..@apache.org
Subject [lucene-solr] branch master updated: SOLR-13420 Routed Aliases now use collection properties instead of core properties
Date Fri, 14 Jun 2019 17:51:42 GMT
This is an automated email from the ASF dual-hosted git repository.

gus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new 5d550a3  SOLR-13420 Routed Aliases now use collection properties instead of core
properties
5d550a3 is described below

commit 5d550a34a9b54d317e51d777d5888b1520767c5a
Author: Gus Heck <gus@apache.org>
AuthorDate: Fri Jun 14 13:51:16 2019 -0400

    SOLR-13420 Routed Aliases now use collection properties instead of core properties
---
 solr/CHANGES.txt                                   |  6 +++++
 .../solr/cloud/api/collections/AliasCmd.java       |  7 ++++-
 .../processor/RoutedAliasUpdateProcessor.java      | 30 +++++++++++++++++++---
 .../TimeRoutedAliasUpdateProcessorTest.java        | 21 ++++++++++++---
 4 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index de56036..83c949c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -78,6 +78,12 @@ Upgrade Notes
 
 * SOLR-13541: Upgrade Jetty to 9.4.19.v20190610. (Erick Erickson, Cao Manh Dat)
 
+* SOLR-13420: Routed Aliases now use collection properties rather than core properties to
identify collections that
+  belong to the alias by default. This should be invisible and fully backwards compatible
from within solr, and
+  existing routed alias collections with core based properties will continue to work, but
new collections created will
+  not add a property to core.properties anymoore so any external code that inspected core.properties
will not find the
+  'routedAliasName' key in new cores belonging to routed aliases.
+
 New Features
 ----------------------
 
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
index 05cca40..752bc5b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AliasCmd.java
@@ -25,6 +25,7 @@ import org.apache.solr.cloud.Overseer;
 import org.apache.solr.cloud.OverseerSolrResponse;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.CollectionProperties;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CollectionParams;
@@ -64,7 +65,6 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd
{
           "We require an explicit " + COLL_CONF);
     }
     createReqParams.set(NAME, createCollName);
-    createReqParams.set("property." + ROUTED_ALIAS_NAME_CORE_PROP, aliasName);
     // a CollectionOperation reads params and produces a message (Map) that is supposed to
be sent to the Overseer.
     //   Although we could create the Map without it, there are a fair amount of rules we
don't want to reproduce.
     final Map<String, Object> createMsgMap = CollectionsHandler.CollectionOperation.CREATE_OP.execute(
@@ -88,6 +88,11 @@ abstract class AliasCmd implements OverseerCollectionMessageHandler.Cmd
{
 
     CollectionsHandler.waitForActiveCollection(createCollName, ocmh.overseer.getCoreContainer(),
         new OverseerSolrResponse(results));
+    CollectionProperties collectionProperties = new CollectionProperties(ocmh.zkStateReader.getZkClient());
+    collectionProperties.setCollectionProperty(createCollName,ROUTED_ALIAS_NAME_CORE_PROP,aliasName);
+    while (!ocmh.zkStateReader.getCollectionProperties(createCollName,1000).containsKey(ROUTED_ALIAS_NAME_CORE_PROP))
{
+      Thread.sleep(50);
+    }
     return results;
   }
 
diff --git a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
index 13dd731..384fcd8 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/RoutedAliasUpdateProcessor.java
@@ -24,6 +24,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.solr.cloud.CloudDescriptor;
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.cloud.api.collections.RoutedAlias;
 import org.apache.solr.common.SolrException;
@@ -33,10 +35,12 @@ import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.schema.SchemaField;
@@ -67,6 +71,11 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor
{
   private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM;
// param
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  // make sure we don't request collection properties any more frequently than once a minute
during
+  // slow continuous indexing, and even less frequently during bulk indexing. (cache is updated
by zk
+  // watch instead of re-requested until indexing has been stopped for the duration specified
here)
+  public static final int CACHE_FOR_MILLIS = 60000;
+
   // refs to std infrastructure
   private final SolrQueryRequest req;
   private final SolrCmdDistributor cmdDistrib;
@@ -79,9 +88,24 @@ public class RoutedAliasUpdateProcessor extends UpdateRequestProcessor
{
 
 
   public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor
next) {
-    //TODO get from "Collection property"
-    final String aliasName = req.getCore().getCoreDescriptor()
-        .getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
+    String aliasName = null;
+    // Demeter please don't arrest us... hide your eyes :(
+    // todo: a core should have a more direct way of finding a collection name, and the collection
properties
+    SolrCore core = req.getCore();
+    CoreDescriptor coreDescriptor = core.getCoreDescriptor();
+    CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
+    if (cloudDescriptor != null) {
+      String collectionName = cloudDescriptor.getCollectionName();
+      CoreContainer coreContainer = core.getCoreContainer();
+      ZkController zkController = coreContainer.getZkController();
+      ZkStateReader zkStateReader = zkController.getZkStateReader();
+      Map<String, String> collectionProperties = zkStateReader.getCollectionProperties(collectionName,
CACHE_FOR_MILLIS);
+      aliasName = collectionProperties.get(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP);
+    }
+    // fall back on core properties (legacy)
+    if (StringUtils.isBlank(aliasName)) {
+      aliasName = coreDescriptor.getCoreProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP,
null);
+    }
     final DistribPhase shardDistribPhase =
         DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
     final DistribPhase aliasDistribPhase =
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
index c1d25a6..6636bb6 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 
@@ -36,13 +37,13 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
 import org.apache.solr.client.solrj.response.FieldStatsInfo;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.UpdateResponse;
-import org.apache.solr.cloud.api.collections.RoutedAlias;
 import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.Utils;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.update.UpdateCommand;
@@ -54,6 +55,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.solr.cloud.api.collections.RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTIONS_ZKNODE;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROPS_ZKNODE;
 
 @LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059")
 public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest {
@@ -98,7 +102,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
     final String col23rd = alias + "_2017-10-23";
     CollectionAdminRequest.createCollection(col23rd, configName, 2, 2)
         .setMaxShardsPerNode(2)
-        .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+        .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
         .process(solrClient);
 
     cluster.waitForActiveCollection(col23rd, 2, 4);
@@ -132,7 +136,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
     //   destined for this collection, Solr will see it already exists and add it to the
alias.
     final String col24th = alias + "_2017-10-24";
     CollectionAdminRequest.createCollection(col24th, configName,  1, 1) // more shards and
replicas now
-        .withProperty(RoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, alias)
+        .withProperty(ROUTED_ALIAS_NAME_CORE_PROP, alias)
         .process(solrClient);
 
     // index 3 documents in a random fashion
@@ -175,6 +179,17 @@ public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcess
     );
     assertInvariants(alias + "_2017-10-26", alias + "_2017-10-25", col24th);
 
+    // verify that collection properties are set when the collections are created. Note:
first 2 collections in
+    // this test have a core property instead, of a collection property but that MUST continue
to work as well
+    // for back compatibility's reasons.
+    Thread.sleep(1000);
+    byte[] data = cluster.getZkClient()
+        .getData(COLLECTIONS_ZKNODE + "/" + alias + "_2017-10-26" + "/" + COLLECTION_PROPS_ZKNODE,null,
null, true);
+    assertNotNull(data);
+    assertTrue(data.length > 0);
+    Map<String,String> props = (Map<String, String>) Utils.fromJSON(data);
+    assertTrue(props.containsKey(ROUTED_ALIAS_NAME_CORE_PROP));
+    assertEquals(alias,props.get(ROUTED_ALIAS_NAME_CORE_PROP));
 
     // update metadata to auto-delete oldest collections
     CollectionAdminRequest.setAliasProperty(alias)


Mime
View raw message