Return-Path: X-Original-To: apmail-lucene-commits-archive@www.apache.org Delivered-To: apmail-lucene-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A8702D1BD for ; Fri, 26 Oct 2012 00:30:49 +0000 (UTC) Received: (qmail 24765 invoked by uid 500); 26 Oct 2012 00:30:49 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 24755 invoked by uid 99); 26 Oct 2012 00:30:49 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Oct 2012 00:30:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 Oct 2012 00:30:47 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8A48123888FE; Fri, 26 Oct 2012 00:30:04 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1402360 - in /lucene/dev/trunk/solr: CHANGES.txt solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java Date: Fri, 26 Oct 2012 00:30:04 -0000 To: commits@lucene.apache.org From: markrmiller@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121026003004.8A48123888FE@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: markrmiller Date: Fri Oct 26 00:30:03 2012 New Revision: 1402360 URL: http://svn.apache.org/viewvc?rev=1402360&view=rev Log: SOLR-3920: Fix server list caching in CloudSolrServer when using more than one collection list with the same instance. Modified: lucene/dev/trunk/solr/CHANGES.txt lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java Modified: lucene/dev/trunk/solr/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1402360&r1=1402359&r2=1402360&view=diff ============================================================================== --- lucene/dev/trunk/solr/CHANGES.txt (original) +++ lucene/dev/trunk/solr/CHANGES.txt Fri Oct 26 00:30:03 2012 @@ -104,6 +104,9 @@ Bug Fixes * SOLR-3981: Fixed bug that resulted in document boosts being compounded in destination fields. (hossman) +* SOLR-3920: Fix server list caching in CloudSolrServer when using more than one + collection list with the same instance. (Grzegorz Sobczyk, Mark Miller) + Other Changes ---------------------- Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1402360&r1=1402359&r2=1402360&view=diff ============================================================================== --- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original) +++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Fri Oct 26 00:30:03 2012 @@ -63,11 +63,12 @@ public class CloudSolrServer extends Sol private HttpClient myClient; Random rand = new Random(); + private Object cachLock = new Object(); // since the state shouldn't change often, should be very cheap reads - private volatile List urlList; - - private volatile List leaderUrlList; - private volatile List replicasList; + private Map> urlLists = new HashMap>(); + private Map> leaderUrlLists = new HashMap>(); + + private Map> replicasLists = new HashMap>(); private volatile int lastClusterStateHashCode; @@ -201,54 +202,67 @@ public class CloudSolrServer extends Sol Set liveNodes = clusterState.getLiveNodes(); - if (sendToLeaders && leaderUrlList == null || !sendToLeaders && urlList == null || clusterState.hashCode() != this.lastClusterStateHashCode) { - - // build a map of unique nodes - // TODO: allow filtering by group, role, etc - Map nodes = new HashMap(); - List urlList = new ArrayList(); - for (Slice slice : slices.values()) { - for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { - ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); - String node = coreNodeProps.getNodeName(); - if (!liveNodes.contains(coreNodeProps.getNodeName()) - || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue; - if (nodes.put(node, nodeProps) == null) { - if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) { - String url = coreNodeProps.getCoreUrl(); - urlList.add(url); - } else if (sendToLeaders) { - String url = coreNodeProps.getCoreUrl(); - replicas.add(url); + List theUrlList; + synchronized (cachLock) { + System.out.println("work with collection:" + collection); + List leaderUrlList = leaderUrlLists.get(collection); + List urlList = urlLists.get(collection); + List replicasList = replicasLists.get(collection); + + if ((sendToLeaders && leaderUrlList == null) || (!sendToLeaders + && urlList == null) + || clusterState.hashCode() != this.lastClusterStateHashCode) { + System.out.println("build a new map for " + collection); + // build a map of unique nodes + // TODO: allow filtering by group, role, etc + Map nodes = new HashMap(); + List urlList2 = new ArrayList(); + for (Slice slice : slices.values()) { + for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) { + ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps); + String node = coreNodeProps.getNodeName(); + if (!liveNodes.contains(coreNodeProps.getNodeName()) + || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue; + if (nodes.put(node, nodeProps) == null) { + if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) { + String url = coreNodeProps.getCoreUrl(); + urlList2.add(url); + } else if (sendToLeaders) { + String url = coreNodeProps.getCoreUrl(); + replicas.add(url); + } } } } + if (sendToLeaders) { + this.leaderUrlLists.put(collection, urlList2); + leaderUrlList = urlList2; + this.replicasLists.put(collection, replicas); + replicasList = replicas; + } else { + this.urlLists.put(collection, urlList2); + urlList = urlList2; + } + this.lastClusterStateHashCode = clusterState.hashCode(); } + if (sendToLeaders) { - this.leaderUrlList = urlList; - this.replicasList = replicas; + theUrlList = new ArrayList(leaderUrlList.size()); + theUrlList.addAll(leaderUrlList); } else { - this.urlList = urlList; + theUrlList = new ArrayList(urlList.size()); + theUrlList.addAll(urlList); + } + Collections.shuffle(theUrlList, rand); + if (sendToLeaders) { + ArrayList theReplicas = new ArrayList( + replicasList.size()); + theReplicas.addAll(replicasList); + Collections.shuffle(theReplicas, rand); + // System.out.println("leaders:" + theUrlList); + // System.out.println("replicas:" + theReplicas); + theUrlList.addAll(theReplicas); } - this.lastClusterStateHashCode = clusterState.hashCode(); - } - - List theUrlList; - if (sendToLeaders) { - theUrlList = new ArrayList(leaderUrlList.size()); - theUrlList.addAll(leaderUrlList); - } else { - theUrlList = new ArrayList(urlList.size()); - theUrlList.addAll(urlList); - } - Collections.shuffle(theUrlList, rand); - if (sendToLeaders) { - ArrayList theReplicas = new ArrayList(replicasList.size()); - theReplicas.addAll(replicasList); - Collections.shuffle(theReplicas, rand); - // System.out.println("leaders:" + theUrlList); - // System.out.println("replicas:" + theReplicas); - theUrlList.addAll(theReplicas); } // System.out.println("########################## MAKING REQUEST TO " + theUrlList); @@ -276,15 +290,19 @@ public class CloudSolrServer extends Sol return lbServer; } - List getUrlList() { - return urlList; + // for tests + Map> getUrlLists() { + return urlLists; } - List getLeaderUrlList() { - return leaderUrlList; + //for tests + Map> getLeaderUrlLists() { + return leaderUrlLists; } - List getReplicasList() { - return replicasList; + //for tests + Map> getReplicasLists() { + return replicasLists; } + } Modified: lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java?rev=1402360&r1=1402359&r2=1402360&view=diff ============================================================================== --- lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java (original) +++ lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java Fri Oct 26 00:30:03 2012 @@ -25,6 +25,7 @@ import java.util.Set; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.AbstractZkTestCase; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.util.ExternalPaths; import org.junit.After; import org.junit.AfterClass; @@ -85,6 +86,7 @@ public class CloudSolrServerTest extends @Override public void doTest() throws Exception { + assertNotNull(cloudClient); handle.clear(); handle.put("QTime", SKIPVAL); @@ -99,9 +101,9 @@ public class CloudSolrServerTest extends // compare leaders list CloudJettyRunner shard1Leader = shardToLeaderJetty.get("shard1"); CloudJettyRunner shard2Leader = shardToLeaderJetty.get("shard2"); - assertEquals(2, cloudClient.getLeaderUrlList().size()); + assertEquals(2, cloudClient.getLeaderUrlLists().get("collection1").size()); HashSet leaderUrlSet = new HashSet(); - leaderUrlSet.addAll(cloudClient.getLeaderUrlList()); + leaderUrlSet.addAll(cloudClient.getLeaderUrlLists().get("collection1")); assertTrue("fail check for leader:" + shard1Leader.url + " in " + leaderUrlSet, leaderUrlSet.contains(shard1Leader.url + "/")); assertTrue("fail check for leader:" + shard2Leader.url + " in " @@ -120,9 +122,9 @@ public class CloudSolrServerTest extends replicas.remove(shard1Leader.url); replicas.remove(shard2Leader.url); - assertEquals(replicas.size(), cloudClient.getReplicasList().size()); + assertEquals(replicas.size(), cloudClient.getReplicasLists().get("collection1").size()); - for (String url : cloudClient.getReplicasList()) { + for (String url : cloudClient.getReplicasLists().get("collection1")) { assertTrue("fail check for replica:" + url + " in " + replicas, replicas.contains(stripTrailingSlash(url))); } @@ -135,5 +137,17 @@ public class CloudSolrServerTest extends } return url; } + + + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = getDoc(fields); + indexDoc(doc); + } + + SolrInputDocument getDoc(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + return doc; + } }