lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject [1/4] lucene-solr:jira/solr-9835: First commit
Date Tue, 07 Mar 2017 07:01:05 GMT
Repository: lucene-solr
Updated Branches:
  refs/heads/jira/solr-9835 [created] 10774b20c


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
index 164eeab..d37cbc5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
@@ -52,6 +52,7 @@ import org.junit.Test;
 public class TestCloudRecovery extends SolrCloudTestCase {
 
   private static final String COLLECTION = "collection1";
+  private static boolean onlyLeaderIndexes;
 
   @BeforeClass
   public static void setupCluster() throws Exception {
@@ -63,8 +64,10 @@ public class TestCloudRecovery extends SolrCloudTestCase {
         .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .configure();
 
+    onlyLeaderIndexes = random().nextBoolean();
     CollectionAdminRequest
         .createCollection(COLLECTION, "config", 2, 2)
+        .setRealtimeReplicas(onlyLeaderIndexes? 1: -1)
         .setMaxShardsPerNode(2)
         .process(cluster.getSolrClient());
     AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(),
@@ -107,7 +110,11 @@ public class TestCloudRecovery extends SolrCloudTestCase {
     resp = cloudClient.query(COLLECTION, params);
     assertEquals(4, resp.getResults().getNumFound());
     // Make sure all nodes is recover from tlog
-    assertEquals(4, countReplayLog.get());
+    if (onlyLeaderIndexes) {
+      assertEquals(2, countReplayLog.get());
+    } else {
+      assertEquals(4, countReplayLog.get());
+    }
 
     // check metrics
     int replicationCount = 0;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
index 8905077..8fbfee3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
@@ -60,7 +60,10 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
   @ShardsFixed(num = 2)
   public void test() throws Exception {
     try (CloudSolrClient client = createCloudClient(null)) {
-      createCollection(null, COLLECTION_NAME, 2, 2, 2, client, null, "conf1");
+      CollectionAdminRequest.Create req = CollectionAdminRequest.createCollection(COLLECTION_NAME,
"conf1",2,2);
+      req.setRealtimeReplicas(1);
+      req.setMaxShardsPerNode(2);
+      client.request(req);
       createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
     }
 
@@ -170,6 +173,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
       Map<String, Object> collection = (Map<String, Object>) collections.get(COLLECTION_NAME);
       assertNotNull(collection);
       assertEquals("conf1", collection.get("configName"));
+      assertEquals("1", collection.get("realtimeReplicas"));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
index 9136d73..ccc09cb 100644
--- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
+++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
@@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory;
 @Slow
 public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private final boolean onlyLeaderIndexes = random().nextBoolean();
 
   @BeforeClass
   public static void beforeSuperClass() throws Exception {
@@ -91,7 +92,12 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase
{
     assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxDocs);
     assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoSoftCommmitMaxDocs);
   }
-  
+
+  @Override
+  protected int getRealtimeReplicas() {
+    return onlyLeaderIndexes? 1 : -1;
+  }
+
   @After
   public void after() {
     System.clearProperty("solr.tests.intClassName");
@@ -179,6 +185,10 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase
{
 
   // The following should work: full update to doc 0, in-place update for doc 0, delete doc
0
   private void outOfOrderDBQsTest() throws Exception {
+    if (onlyLeaderIndexes) {
+      log.info("RTG with DBQs are not working in active replicas");
+      return;
+    }
     
     clearIndex();
     commit();
@@ -245,6 +255,10 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase
{
   }
 
   private void reorderedDBQIndividualReplicaTest() throws Exception {
+    if (onlyLeaderIndexes) {
+      log.info("RTG with DBQs are not working in active replicas");
+      return;
+    }
     clearIndex();
     commit();
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 94750c0a..8beb6ed 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -366,6 +366,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
 
     private Properties properties;
     protected Boolean autoAddReplicas;
+    protected Integer realtimeReplicas;
     protected Integer stateFormat;
     private String[] rule , snitch;
 
@@ -407,6 +408,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
     public Create setNumShards(Integer numShards) {this.numShards = numShards; return this;
}
     public Create setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards;
return this; }
     public Create setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas;
return this; }
+    public Create setRealtimeReplicas(Integer realtimeReplicas) { this.realtimeReplicas =
realtimeReplicas; return this;}
     @Deprecated
     public Create setReplicationFactor(Integer repl) { this.replicationFactor = repl; return
this; }
     public Create setStateFormat(Integer stateFormat) { this.stateFormat = stateFormat; return
this; }
@@ -421,6 +423,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
     public Integer getMaxShardsPerNode() { return maxShardsPerNode; }
     public Integer getReplicationFactor() { return replicationFactor; }
     public Boolean getAutoAddReplicas() { return autoAddReplicas; }
+    public Integer getRealtimeReplicas() { return realtimeReplicas; }
     public Integer getStateFormat() { return stateFormat; }
     
     /**
@@ -507,6 +510,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
       if (autoAddReplicas != null) {
         params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas);
       }
+      if (realtimeReplicas != null) {
+        params.set(ZkStateReader.REALTIME_REPLICAS, realtimeReplicas);
+      }
       if(properties != null) {
         addProperties(params, properties);
       }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
index 179b9d5..bf0f04f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
@@ -33,6 +33,7 @@ import org.noggit.JSONWriter;
 
 import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
 import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.common.cloud.ZkStateReader.REALTIME_REPLICAS;
 import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
 
 /**
@@ -59,6 +60,7 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice>
{
   private final Integer replicationFactor;
   private final Integer maxShardsPerNode;
   private final Boolean autoAddReplicas;
+  private final Integer realtimeReplicas;
 
 
   public DocCollection(String name, Map<String, Slice> slices, Map<String, Object>
props, DocRouter router) {
@@ -84,6 +86,11 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice>
{
     this.maxShardsPerNode = (Integer) verifyProp(props, MAX_SHARDS_PER_NODE);
     Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS);
     this.autoAddReplicas = autoAddReplicas == null ? false : autoAddReplicas;
+    Integer realtimeReplicas = (Integer) verifyProp(props, REALTIME_REPLICAS);
+    this.realtimeReplicas = realtimeReplicas == null ? -1 : realtimeReplicas;
+    if (this.realtimeReplicas != -1 && this.realtimeReplicas != 1) {
+      throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid realtimeReplicas must be 1
or -1, found:" + this.realtimeReplicas);
+    }
     verifyProp(props, RULE);
     verifyProp(props, SNITCH);
     Iterator<Map.Entry<String, Slice>> iter = slices.entrySet().iterator();
@@ -126,6 +133,7 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice>
{
     switch (propName) {
       case MAX_SHARDS_PER_NODE:
       case REPLICATION_FACTOR:
+      case REALTIME_REPLICAS:
         return Integer.parseInt(o.toString());
       case AUTO_ADD_REPLICAS:
         return Boolean.parseBoolean(o.toString());
@@ -226,6 +234,10 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice>
{
     return maxShardsPerNode;
   }
 
+  public int getRealtimeReplicas() {
+    return realtimeReplicas;
+  }
+
   public String getZNode(){
     return znode;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index fea5978..51b4b59 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -96,6 +96,7 @@ public class ZkStateReader implements Closeable {
   public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
   public static final String AUTO_ADD_REPLICAS = "autoAddReplicas";
   public static final String MAX_CORES_PER_NODE = "maxCoresPerNode";
+  public static final String REALTIME_REPLICAS = "realtimeReplicas";
 
   public static final String ROLES = "/roles.json";
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index 04eb722..ade1c69 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -272,6 +272,10 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
     chaosMonkey = new ChaosMonkey(zkServer, zkStateReader, DEFAULT_COLLECTION,
         shardToJetty, shardToLeaderJetty);
   }
+
+  protected int getRealtimeReplicas() {
+    return -1;
+  }
   
   protected CloudSolrClient createCloudClient(String defaultCollection) {
     CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean());
@@ -383,7 +387,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
           Utils.toJSON(Utils.makeMap(Overseer.QUEUE_OPERATION,
               CollectionParams.CollectionAction.CREATE.toLower(), "name",
               DEFAULT_COLLECTION, "numShards", String.valueOf(sliceCount),
-              DocCollection.STATE_FORMAT, getStateFormat())));
+              DocCollection.STATE_FORMAT, getStateFormat(),
+              ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas())));
       zkClient.close();
     }
 
@@ -1619,7 +1624,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
         NUM_SLICES, numShards,
         ZkStateReader.REPLICATION_FACTOR, replicationFactor,
         CREATE_NODE_SET, createNodeSetStr,
-        ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode),
+        ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode,
+        ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas()),
         client);
   }
 
@@ -1631,7 +1637,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
         NUM_SLICES, numShards,
         ZkStateReader.REPLICATION_FACTOR, replicationFactor,
         CREATE_NODE_SET, createNodeSetStr,
-        ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode),
+        ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode,
+        ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas()),
         client, configName);
   }
 
@@ -1814,6 +1821,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
     Map<String, Object> props = makeMap(
         ZkStateReader.REPLICATION_FACTOR, replicationFactor,
         ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode,
+        ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas(),
         NUM_SLICES, numShards);
     Map<String,List<Integer>> collectionInfos = new HashMap<>();
     createCollection(collectionInfos, collName, props, client);


Mime
View raw message