Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B7931200C3F for ; Tue, 7 Mar 2017 08:01:07 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B632C160B82; Tue, 7 Mar 2017 07:01:07 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B8F55160B68 for ; Tue, 7 Mar 2017 08:01:06 +0100 (CET) Received: (qmail 41106 invoked by uid 500); 7 Mar 2017 07:01:05 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 41093 invoked by uid 99); 7 Mar 2017 07:01:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Mar 2017 07:01:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6B78CDFDE6; Tue, 7 Mar 2017 07:01:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: datcm@apache.org To: commits@lucene.apache.org Date: Tue, 07 Mar 2017 07:01:05 -0000 Message-Id: <4db35b05308d4111a11d372ad0c6c9ff@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/4] lucene-solr:jira/solr-9835: First commit archived-at: Tue, 07 Mar 2017 07:01:07 -0000 Repository: lucene-solr Updated Branches: refs/heads/jira/solr-9835 [created] 10774b20c http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java index 164eeab..d37cbc5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java @@ -52,6 +52,7 @@ import org.junit.Test; public class TestCloudRecovery extends SolrCloudTestCase { private static final String COLLECTION = "collection1"; + private static boolean onlyLeaderIndexes; @BeforeClass public static void setupCluster() throws Exception { @@ -63,8 +64,10 @@ public class TestCloudRecovery extends SolrCloudTestCase { .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) .configure(); + onlyLeaderIndexes = random().nextBoolean(); CollectionAdminRequest .createCollection(COLLECTION, "config", 2, 2) + .setRealtimeReplicas(onlyLeaderIndexes? 1: -1) .setMaxShardsPerNode(2) .process(cluster.getSolrClient()); AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), @@ -107,7 +110,11 @@ public class TestCloudRecovery extends SolrCloudTestCase { resp = cloudClient.query(COLLECTION, params); assertEquals(4, resp.getResults().getNumFound()); // Make sure all nodes is recover from tlog - assertEquals(4, countReplayLog.get()); + if (onlyLeaderIndexes) { + assertEquals(2, countReplayLog.get()); + } else { + assertEquals(4, countReplayLog.get()); + } // check metrics int replicationCount = 0; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java index 8905077..8fbfee3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java @@ -60,7 +60,10 @@ public class TestCollectionAPI extends ReplicaPropertiesBase { @ShardsFixed(num = 2) public void test() throws Exception { try (CloudSolrClient client = createCloudClient(null)) { - createCollection(null, COLLECTION_NAME, 2, 2, 2, client, null, "conf1"); + CollectionAdminRequest.Create req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2,2); + req.setRealtimeReplicas(1); + req.setMaxShardsPerNode(2); + client.request(req); createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1"); } @@ -170,6 +173,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase { Map collection = (Map) collections.get(COLLECTION_NAME); assertNotNull(collection); assertEquals("conf1", collection.get("configName")); + assertEquals("1", collection.get("realtimeReplicas")); } } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java ---------------------------------------------------------------------- diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java index 9136d73..ccc09cb 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java @@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory; @Slow public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final boolean onlyLeaderIndexes = random().nextBoolean(); @BeforeClass public static void beforeSuperClass() throws Exception { @@ -91,7 +92,12 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoCommmitMaxDocs); assertEquals(-1, h.getCore().getSolrConfig().getUpdateHandlerInfo().autoSoftCommmitMaxDocs); } - + + @Override + protected int getRealtimeReplicas() { + return onlyLeaderIndexes? 1 : -1; + } + @After public void after() { System.clearProperty("solr.tests.intClassName"); @@ -179,6 +185,10 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { // The following should work: full update to doc 0, in-place update for doc 0, delete doc 0 private void outOfOrderDBQsTest() throws Exception { + if (onlyLeaderIndexes) { + log.info("RTG with DBQs are not working in active replicas"); + return; + } clearIndex(); commit(); @@ -245,6 +255,10 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase { } private void reorderedDBQIndividualReplicaTest() throws Exception { + if (onlyLeaderIndexes) { + log.info("RTG with DBQs are not working in active replicas"); + return; + } clearIndex(); commit(); http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java ---------------------------------------------------------------------- diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index 94750c0a..8beb6ed 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -366,6 +366,7 @@ public abstract class CollectionAdminRequest private Properties properties; protected Boolean autoAddReplicas; + protected Integer realtimeReplicas; protected Integer stateFormat; private String[] rule , snitch; @@ -407,6 +408,7 @@ public abstract class CollectionAdminRequest public Create setNumShards(Integer numShards) {this.numShards = numShards; return this; } public Create setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; return this; } public Create setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; return this; } + public Create setRealtimeReplicas(Integer realtimeReplicas) { this.realtimeReplicas = realtimeReplicas; return this;} @Deprecated public Create setReplicationFactor(Integer repl) { this.replicationFactor = repl; return this; } public Create setStateFormat(Integer stateFormat) { this.stateFormat = stateFormat; return this; } @@ -421,6 +423,7 @@ public abstract class CollectionAdminRequest public Integer getMaxShardsPerNode() { return maxShardsPerNode; } public Integer getReplicationFactor() { return replicationFactor; } public Boolean getAutoAddReplicas() { return autoAddReplicas; } + public Integer getRealtimeReplicas() { return realtimeReplicas; } public Integer getStateFormat() { return stateFormat; } /** @@ -507,6 +510,9 @@ public abstract class CollectionAdminRequest if (autoAddReplicas != null) { params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas); } + if (realtimeReplicas != null) { + params.set(ZkStateReader.REALTIME_REPLICAS, realtimeReplicas); + } if(properties != null) { addProperties(params, properties); } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java ---------------------------------------------------------------------- diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java index 179b9d5..bf0f04f 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java @@ -33,6 +33,7 @@ import org.noggit.JSONWriter; import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; +import static org.apache.solr.common.cloud.ZkStateReader.REALTIME_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; /** @@ -59,6 +60,7 @@ public class DocCollection extends ZkNodeProps implements Iterable { private final Integer replicationFactor; private final Integer maxShardsPerNode; private final Boolean autoAddReplicas; + private final Integer realtimeReplicas; public DocCollection(String name, Map slices, Map props, DocRouter router) { @@ -84,6 +86,11 @@ public class DocCollection extends ZkNodeProps implements Iterable { this.maxShardsPerNode = (Integer) verifyProp(props, MAX_SHARDS_PER_NODE); Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS); this.autoAddReplicas = autoAddReplicas == null ? false : autoAddReplicas; + Integer realtimeReplicas = (Integer) verifyProp(props, REALTIME_REPLICAS); + this.realtimeReplicas = realtimeReplicas == null ? -1 : realtimeReplicas; + if (this.realtimeReplicas != -1 && this.realtimeReplicas != 1) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid realtimeReplicas must be 1 or -1, found:" + this.realtimeReplicas); + } verifyProp(props, RULE); verifyProp(props, SNITCH); Iterator> iter = slices.entrySet().iterator(); @@ -126,6 +133,7 @@ public class DocCollection extends ZkNodeProps implements Iterable { switch (propName) { case MAX_SHARDS_PER_NODE: case REPLICATION_FACTOR: + case REALTIME_REPLICAS: return Integer.parseInt(o.toString()); case AUTO_ADD_REPLICAS: return Boolean.parseBoolean(o.toString()); @@ -226,6 +234,10 @@ public class DocCollection extends ZkNodeProps implements Iterable { return maxShardsPerNode; } + public int getRealtimeReplicas() { + return realtimeReplicas; + } + public String getZNode(){ return znode; } http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java ---------------------------------------------------------------------- diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index fea5978..51b4b59 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -96,6 +96,7 @@ public class ZkStateReader implements Closeable { public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode"; public static final String AUTO_ADD_REPLICAS = "autoAddReplicas"; public static final String MAX_CORES_PER_NODE = "maxCoresPerNode"; + public static final String REALTIME_REPLICAS = "realtimeReplicas"; public static final String ROLES = "/roles.json"; http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c1a5e6fc/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java ---------------------------------------------------------------------- diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 04eb722..ade1c69 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -272,6 +272,10 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes chaosMonkey = new ChaosMonkey(zkServer, zkStateReader, DEFAULT_COLLECTION, shardToJetty, shardToLeaderJetty); } + + protected int getRealtimeReplicas() { + return -1; + } protected CloudSolrClient createCloudClient(String defaultCollection) { CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), random().nextBoolean()); @@ -383,7 +387,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes Utils.toJSON(Utils.makeMap(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), "name", DEFAULT_COLLECTION, "numShards", String.valueOf(sliceCount), - DocCollection.STATE_FORMAT, getStateFormat()))); + DocCollection.STATE_FORMAT, getStateFormat(), + ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas()))); zkClient.close(); } @@ -1619,7 +1624,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes NUM_SLICES, numShards, ZkStateReader.REPLICATION_FACTOR, replicationFactor, CREATE_NODE_SET, createNodeSetStr, - ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode), + ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode, + ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas()), client); } @@ -1631,7 +1637,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes NUM_SLICES, numShards, ZkStateReader.REPLICATION_FACTOR, replicationFactor, CREATE_NODE_SET, createNodeSetStr, - ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode), + ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode, + ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas()), client, configName); } @@ -1814,6 +1821,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes Map props = makeMap( ZkStateReader.REPLICATION_FACTOR, replicationFactor, ZkStateReader.MAX_SHARDS_PER_NODE, maxShardsPerNode, + ZkStateReader.REALTIME_REPLICAS, getRealtimeReplicas(), NUM_SLICES, numShards); Map> collectionInfos = new HashMap<>(); createCollection(collectionInfos, collName, props, client);