lucene-solr-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r900841 - in /lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component: QueryComponent.java ResponseBuilder.java SearchHandler.java ShardRequest.java
Date Tue, 19 Jan 2010 16:52:19 GMT
Author: yonik
Date: Tue Jan 19 16:52:19 2010
New Revision: 900841

URL: http://svn.apache.org/viewvc?rev=900841&view=rev
Log:
draft - distrib search uses CloudState

Modified:
    lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
    lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
    lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java

Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
(original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java
Tue Jan 19 16:52:19 2010
@@ -39,6 +39,10 @@
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.*;
 import org.apache.solr.util.SolrPluginUtils;
+import org.apache.solr.cloud.CloudState;
+import org.apache.solr.cloud.Slice;
+import org.apache.solr.cloud.ZkNodeProps;
+
 
 import java.io.IOException;
 import java.net.URL;
@@ -104,14 +108,78 @@
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
     }
 
-    // TODO: temporary... this should go in a different component.
+    // TODO: temporary... this should go in a different component, or in SearchHandler
+    rb.isDistrib = params.getBool("distrib",false);
     String shards = params.get(ShardParams.SHARDS);
-    // nocommit : get shards based on ZooKeeper info, using load balancing
-    // shards = req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getSearchNodes();
-    if (shards != null) {
-      List<String> lst = StrUtils.splitSmart(shards, ",", true);
-      rb.shards = lst.toArray(new String[lst.size()]);
+
+    // for back compat, a shards param with URLs like localhost:8983/solr will mean that
this
+    // search is distributed.
+    boolean hasShardURL = shards != null && shards.charAt('/') > 0;
+    rb.isDistrib = hasShardURL | rb.isDistrib;  
+
+    if (rb.isDistrib) {
+      CloudState cloudState = null;
+
+      if (shards != null) {
+        List<String> lst = StrUtils.splitSmart(shards, ",", true);
+        rb.shards = lst.toArray(new String[lst.size()]);
+        rb.slices = new String[rb.shards.length];
+        for (int i=0; i<rb.shards.length; i++) {
+          if (rb.shards[i].indexOf('/') < 0) {
+            // this is a logical shard
+            rb.slices[i] = rb.shards[i];
+            rb.shards[i] = null;
+          }
+        }
+      } else {
+        // we weren't provided with a list of slices to query, so find the list that will
cover the complete index
+
+        cloudState =  req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getCloudInfo();
+
+        // TODO: EXAMPLE
+        rb.slices = new String[]{"shard1","shard2"};
+        rb.shards = new String[rb.shards.length];
+      }
+
+      //
+      // Map slices to shards
+      //
+      for (int i=0; i<rb.shards.length; i++) {
+        if (rb.shards[i] == null) {
+          if (cloudState == null) {
+              cloudState =  req.getCore().getCoreDescriptor().getCoreContainer().getZooKeeperController().getCloudInfo();
+          }
+          String sliceStr = rb.slices[i];
+          Slice slice = cloudState.getSlice(sliceStr);
+
+          if (slice==null) {
+            // TODO: we could treat this as "all servers down" for a slice if partial results
are enabled.
+            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no such slice:
" + sliceStr);
+          }
+
+          Map<String, ZkNodeProps> sliceShards = slice.getShards();
+
+          // For now, recreate the | delimited list of equivalent servers
+          StringBuilder sliceShardsStr = new StringBuilder();
+          boolean first = true;
+          for (ZkNodeProps nodeProps : sliceShards.values()) {
+            if (first) {
+              first = false;
+            } else {
+              sliceShardsStr.append('|');  
+            }
+            String url = nodeProps.get("url");
+            if (url.startsWith("http://"))
+              url = url.substring(7);
+            sliceShardsStr.append(url);
+          }
+
+          rb.shards[i] = sliceShardsStr.toString();
+        }
+      }
     }
+    
+    
     String shards_rows = params.get(ShardParams.SHARDS_ROWS);
     if(shards_rows != null) {
       rb.shards_rows = Integer.parseInt(shards_rows);

Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
(original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ResponseBuilder.java
Tue Jan 19 16:52:19 2010
@@ -94,7 +94,9 @@
   public int stage;  // What stage is this current request at?
 
   //The address of the Shard
+  boolean isDistrib; // is this a distributed search?
   public String[] shards;
+  public String[] slices; // the optional logical ids of the shards
   public int shards_rows = -1;
   public int shards_start = -1;
   public List<ShardRequest> outgoing;  // requests to be sent

Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
(original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/SearchHandler.java
Tue Jan 19 16:52:19 2010
@@ -189,7 +189,7 @@
       subt.stop();
     }
 
-    if (rb.shards == null) {
+    if (!rb.isDistrib) {
       // a normal non-distributed request
 
       // The semantics of debugging vs not debugging are different enough that
@@ -466,6 +466,13 @@
           // no need to set the response parser as binary is the default
           // req.setResponseParser(new BinaryResponseParser());
 
+          // if there are no shards available for a slice, urls.size()==0
+          if (urls.size()==0) {
+            // TODO: what's the right error code here? We should use the same thing when
+            // all of the servers for a shard are down.
+            throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "no servers
hosting shard");
+          }
+
           if (urls.size() <= 1) {
             String url = urls.get(0);
             srsp.setShardAddress(url);

Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java?rev=900841&r1=900840&r2=900841&view=diff
==============================================================================
--- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
(original)
+++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/ShardRequest.java
Tue Jan 19 16:52:19 2010
@@ -41,8 +41,7 @@
   public int purpose;  // the purpose of this request
 
   public String[] shards;  // the shards this request should be sent to, null for all
-// TODO: how to request a specific shard address?
-
+  public String[] slices;  // the slice of the index requested for each
 
   public ModifiableSolrParams params;
 



Mime
View raw message