lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From no...@apache.org
Subject svn commit: r1513356 - in /lucene/dev/trunk/solr: core/src/java/org/apache/solr/update/ core/src/test/org/apache/solr/cloud/ example/solr/collection1/conf/ solrj/src/java/org/apache/solr/common/cloud/
Date Tue, 13 Aug 2013 05:49:17 GMT
Author: noble
Date: Tue Aug 13 05:49:17 2013
New Revision: 1513356

URL: http://svn.apache.org/r1513356
Log:
SOLR-5017 support for routeField in COmpositeId router also

Modified:
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
    lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
    lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java Tue
Aug 13 05:49:17 2013
@@ -172,7 +172,7 @@ public class SolrIndexSplitter {
 
       int hash = 0;
       if (hashRouter != null) {
-        hash = hashRouter.sliceHash(idString, null, null);
+        hash = hashRouter.sliceHash(idString, null, null, null);
       }
       // int hash = Hash.murmurhash3_x86_32(ref, ref.offset, ref.length, 0);
 

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/CustomCollectionTest.java Tue
Aug 13 05:49:17 2013
@@ -22,6 +22,7 @@ import org.apache.lucene.util.LuceneTest
 import org.apache.lucene.util._TestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -62,6 +63,7 @@ import java.util.concurrent.ThreadPoolEx
 import java.util.concurrent.TimeUnit;
 
 import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
 import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
 import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
 import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
@@ -132,6 +134,7 @@ public class CustomCollectionTest extend
   @Override
   public void doTest() throws Exception {
     testCustomCollectionsAPI();
+    testRouteFieldForHashRouter();
     if (DEBUG) {
       super.printLayout();
     }
@@ -242,8 +245,8 @@ public class CustomCollectionTest extend
     collectionClient.commit();
 
     assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","b")).getResults().getNumFound());
-    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","a")).getResults().getNumFound());
+    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"b")).getResults().getNumFound());
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"a")).getResults().getNumFound());
 
     collectionClient.deleteByQuery("*:*");
     collectionClient.commit(true,true);
@@ -263,8 +266,8 @@ public class CustomCollectionTest extend
     collectionClient.request(up);
 
     assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","a")).getResults().getNumFound());
-    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","c")).getResults().getNumFound());
+    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"a")).getResults().getNumFound());
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"c")).getResults().getNumFound());
 
     //Testing CREATESHARD
     ModifiableSolrParams params = new ModifiableSolrParams();
@@ -292,7 +295,7 @@ public class CustomCollectionTest extend
     collectionClient.add(getDoc(id, 66, i1, -600, tlong, 600, t1,
         "humpty dumpy sat on a wall", _ROUTE_,"x"));
     collectionClient.commit();
-    assertEquals(1, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","x")).getResults().getNumFound());
+    assertEquals(1, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"x")).getResults().getNumFound());
 
 
     int numShards = 4;
@@ -349,9 +352,67 @@ public class CustomCollectionTest extend
     collectionClient.commit();
 
     assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
-    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys","b")).getResults().getNumFound());
+    assertEquals(0, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_,"b")).getResults().getNumFound());
     //TODO debug the following case
-    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam("shard.keys", "a")).getResults().getNumFound());
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_, "a")).getResults().getNumFound());
+
+
+  }
+
+  private void testRouteFieldForHashRouter()throws Exception{
+    String collectionName = "routeFieldColl";
+    int numShards = 4;
+    int replicationFactor = 2;
+    int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrServer()
+        .getZkStateReader().getClusterState().getLiveNodes().size())) + 1;
+
+    HashMap<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
+    CloudSolrServer client = null;
+    String shard_fld = "shard_s";
+    try {
+      client = createCloudClient(null);
+      Map<String, Object> props = OverseerCollectionProcessor.asMap(
+          REPLICATION_FACTOR, replicationFactor,
+          MAX_SHARDS_PER_NODE, maxShardsPerNode,
+          NUM_SLICES,numShards,
+          DocRouter.ROUTE_FIELD, shard_fld);
+
+      createCollection(collectionInfos, collectionName,props,client);
+    } finally {
+      if (client != null) client.shutdown();
+    }
+
+    List<Integer> list = collectionInfos.get(collectionName);
+    checkForCollection(collectionName, list, null);
+
+
+    String url = getUrlFromZk(collectionName);
+
+    HttpSolrServer collectionClient = new HttpSolrServer(url);
+
+    // poll for a second - it can take a moment before we are ready to serve
+    waitForNon403or404or503(collectionClient);
+
+
+    collectionClient = new HttpSolrServer(url);
+
+
+    // lets try and use the solrj client to index a couple documents
+
+    collectionClient.add(getDoc(id, 6, i1, -600, tlong, 600, t1,
+        "humpty dumpy sat on a wall", shard_fld,"a"));
+
+    collectionClient.add(getDoc(id, 7, i1, -600, tlong, 600, t1,
+        "humpty dumpy3 sat on a walls", shard_fld,"a"));
+
+    collectionClient.add(getDoc(id, 8, i1, -600, tlong, 600, t1,
+        "humpty dumpy2 sat on a walled", shard_fld,"a"));
+
+    collectionClient.commit();
+
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
+    //TODO debug the following case
+    assertEquals(3, collectionClient.query(new SolrQuery("*:*").setParam(_ROUTE_, "a")).getResults().getNumFound());
 
 
   }

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java Tue Aug
13 05:49:17 2013
@@ -17,15 +17,6 @@ package org.apache.solr.cloud;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
 import org.apache.http.params.CoreConnectionPNames;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
@@ -50,6 +41,15 @@ import org.apache.solr.update.DirectUpda
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
 public class ShardSplitTest extends BasicDistributedZkTest {
 
   public static final String SHARD1_0 = SHARD1 + "_0";
@@ -289,7 +289,7 @@ public class ShardSplitTest extends Basi
     int hash = 0;
     if (router instanceof HashBasedRouter) {
       HashBasedRouter hashBasedRouter = (HashBasedRouter) router;
-      hash = hashBasedRouter.sliceHash(id, null, null);
+      hash = hashBasedRouter.sliceHash(id, null, null,null);
     }
     for (int i = 0; i < ranges.size(); i++) {
       DocRouter.Range range = ranges.get(i);

Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/solrconfig.xml Tue Aug 13 05:49:17
2013
@@ -1480,6 +1480,7 @@
   <searchComponent name="terms" class="solr.TermsComponent"/>
 
   <!-- A request handler for demonstrating the terms component -->
+  <requestHandler name="/js" class="org.apache.solr.handler.js.JavaScriptRequestHandler"
startup="lazy"/>
   <requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
      <lst name="defaults">
       <bool name="terms">true</bool>

Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
(original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java
Tue Aug 13 05:49:17 2013
@@ -17,6 +17,7 @@ package org.apache.solr.common.cloud;
  * limitations under the License.
  */
 
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.Hash;
@@ -59,17 +60,33 @@ public class CompositeIdRouter extends H
   }
 
   @Override
-  public int sliceHash(String id, SolrInputDocument doc, SolrParams params) {
-    int idx = id.indexOf(separator);
-    if (idx < 0) {
-      return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+  public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection
collection) {
+    String shardFieldName = collection ==null? null: collection.getStr(DocRouter.ROUTE_FIELD);
+    String part1 = null;
+    int idx = 0;
+    int commaIdx = 0;
+
+    if(shardFieldName == null || doc == null) {
+      idx = id.indexOf(separator);
+      if (idx < 0) {
+        return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+      }
+      part1 = id.substring(0, idx);
+      commaIdx = part1.indexOf(bitsSeparator);
+
+    } else {
+      Object o = doc.getFieldValue(shardFieldName);
+      if (o != null) {
+        part1 = o.toString();
+        return Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
+      } else {
+        throw new SolrException (SolrException.ErrorCode.BAD_REQUEST, "No value for :"+shardFieldName
+ ". Unable to identify shard");
+      }
     }
 
     int m1 = mask1;
     int m2 = mask2;
 
-    String part1 = id.substring(0,idx);
-    int commaIdx = part1.indexOf(bitsSeparator);
     if (commaIdx > 0) {
       int firstBits = getBits(part1, commaIdx);
       if (firstBits >= 0) {

Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java?rev=1513356&r1=1513355&r2=1513356&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java
(original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/HashBasedRouter.java
Tue Aug 13 05:49:17 2013
@@ -30,19 +30,19 @@ public abstract class HashBasedRouter ex
   @Override
   public Slice getTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, DocCollection
collection) {
     if (id == null) id = getId(sdoc, params);
-    int hash = sliceHash(id, sdoc, params);
+    int hash = sliceHash(id, sdoc, params,collection);
     return hashToSlice(hash, collection);
   }
 
   @Override
   public boolean isTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, String
shardId, DocCollection collection) {
     if (id == null) id = getId(sdoc, params);
-    int hash = sliceHash(id, sdoc, params);
+    int hash = sliceHash(id, sdoc, params, collection);
     Range range = collection.getSlice(shardId).getRange();
     return range != null && range.includes(hash);
   }
 
-  public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params) {
+  public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params, DocCollection
collection) {
     return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
   }
 



Mime
View raw message