lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1450015 - in /lucene/dev/trunk/solr: CHANGES.txt core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
Date Tue, 26 Feb 2013 04:29:57 GMT
Author: markrmiller
Date: Tue Feb 26 04:29:56 2013
New Revision: 1450015

URL: http://svn.apache.org/r1450015
Log:
SOLR-4210: Requests to a Collection that does not exist on the receiving node should be proxied
to a suitable node.

Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1450015&r1=1450014&r2=1450015&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Feb 26 04:29:56 2013
@@ -91,6 +91,9 @@ New Features
 * SOLR-4078: Allow custom naming of SolrCloud nodes so that a new host:port 
   combination can take over for a previous shard. (Mark Miller)
 
+* SOLR-4210: Requests to a Collection that does not exist on the receiving node
+  should be proxied to a suitable node. (Mark Miller, Po Rui)
+
 Bug Fixes
 ----------------------
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java?rev=1450015&r1=1450014&r2=1450015&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java Tue
Feb 26 04:29:56 2013
@@ -17,13 +17,22 @@
 
 package org.apache.solr.servlet;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -33,15 +42,18 @@ import javax.servlet.Filter;
 import javax.servlet.FilterChain;
 import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
 import javax.servlet.ServletRequest;
 import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
@@ -50,7 +62,6 @@ import org.apache.solr.common.params.Sol
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.Config;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
@@ -206,11 +217,22 @@ public class SolrDispatchFilter implemen
           if (core != null) {
             // we found a core, update the path
             path = path.substring( idx );
-          } else {
-            // try the default core
+          }
+          
+          // if we couldn't find it locally, look on other nodes
+          if (core == null && idx > 0) {
+            String coreUrl = getRemotCoreUrl(cores, corename);
+            if (coreUrl != null) {
+              path = path.substring( idx );
+              remoteQuery(coreUrl + path, req, solrReq, resp);
+              return;
+            }
+          }
+          
+          // try the default core
+          if (core == null) {
             core = cores.getCore("");
           }
-          // TODO: if we couldn't find it locally, look on other nodes
         }
 
         // With a valid core...
@@ -305,6 +327,118 @@ public class SolrDispatchFilter implemen
     chain.doFilter(request, response);
   }
   
+  private void remoteQuery(String coreUrl, HttpServletRequest req,
+      SolrQueryRequest solrReq, HttpServletResponse resp) throws IOException {
+    
+    try {
+      String urlstr = coreUrl;
+      
+      String queryString = req.getQueryString();
+      
+      urlstr += queryString == null ? "" : "?" + queryString;
+      
+      URL url = new URL(urlstr);
+      HttpURLConnection con = (HttpURLConnection) url.openConnection();
+      con.setRequestMethod(req.getMethod());
+      con.setUseCaches(true);
+      
+      con.setDoOutput(true);
+      con.setDoInput(true);
+      for (Enumeration e = req.getHeaderNames(); e.hasMoreElements();) {
+        String headerName = e.nextElement().toString();
+        con.setRequestProperty(headerName, req.getHeader(headerName));
+      }
+      try {
+        con.connect();
+        
+        int theByte;
+        if (req.getMethod().equals("POST")) {
+          BufferedInputStream bis = new BufferedInputStream(
+              req.getInputStream());
+          BufferedOutputStream os = new BufferedOutputStream(
+              con.getOutputStream());
+          try {
+            while ((theByte = bis.read()) != -1) {
+              os.write(theByte);
+            }
+            os.flush();
+          } finally {
+            IOUtils.closeQuietly(os);
+            IOUtils.closeQuietly(bis);
+          }
+        }
+        
+        resp.setStatus(con.getResponseCode());
+        
+        for (Iterator i = con.getHeaderFields().entrySet().iterator(); i
+            .hasNext();) {
+          Map.Entry mapEntry = (Map.Entry) i.next();
+          if (mapEntry.getKey() != null) resp.setHeader(mapEntry.getKey()
+              .toString(), ((List) mapEntry.getValue()).get(0).toString());
+        }
+        
+        resp.setCharacterEncoding(con.getContentEncoding());
+        resp.setContentType(con.getContentType());
+        
+        BufferedInputStream bis = new BufferedInputStream(con.getInputStream());
+        ServletOutputStream os = resp.getOutputStream();
+        try {
+          while ((theByte = bis.read()) != -1) {
+            os.write(theByte);
+          }
+          os.flush();
+        } finally {
+          IOUtils.closeQuietly(os);
+          IOUtils.closeQuietly(bis);
+        }
+      } finally {
+        con.disconnect();
+      }
+    } catch (IOException e) {
+      // return exception
+      resp.getWriter().println(e.getStackTrace().toString());
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+          "This node forword query failed: " + coreUrl);
+    }
+    
+  }
+  
+  private String getRemotCoreUrl(CoreContainer cores, String collectionName) {
+    ClusterState clusterState = cores.getZkController().getClusterState();
+    Collection<Slice> slices = clusterState.getSlices(collectionName);
+    
+    if (slices == null) {
+      Set<String> collections = clusterState.getCollections();
+      for (String collection : collections) {
+        slices = new ArrayList<Slice>();
+        slices.addAll(clusterState.getSlices(collection));
+      }
+    }
+
+    Set<String> liveNodes = clusterState.getLiveNodes();
+    Iterator<Slice> it = slices.iterator();
+    while (it.hasNext()) {
+      Slice slice = it.next();
+      Map<String,Replica> sliceShards = slice.getReplicasMap();
+      for (ZkNodeProps nodeProps : sliceShards.values()) {
+        ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
+        if (liveNodes.contains(coreNodeProps.getNodeName())
+            && coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) {
+          String coreUrl = coreNodeProps.getCoreUrl();
+          if (coreUrl.endsWith("/")) {
+            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
+          }
+          if (coreNodeProps.getBaseUrl().equals(cores.getZkController().getBaseUrl())) {
+            // don't count a local core
+            return null;
+          }
+          return coreUrl;
+        }
+      }
+    }
+    return null;
+  }
+  
   private SolrCore getCoreByCollection(CoreContainer cores, String corename, String path)
{
     String collection = corename;
     ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1450015&r1=1450014&r2=1450015&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
(original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
Tue Feb 26 04:29:56 2013
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.Solr
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.CoreAdminRequest.Create;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.QueryResponse;
@@ -55,6 +56,8 @@ import org.junit.BeforeClass;
  * work as expected.
  */
 public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
+  private static final String ONE_NODE_COLLECTION = "onenodecollection";
+
   @BeforeClass
   public static void beforeThisClass2() throws Exception {
     // TODO: we use an fs based dir because something
@@ -79,6 +82,8 @@ public class BasicDistributedZk2Test ext
       handle.put("QTime", SKIPVAL);
       handle.put("timestamp", SKIPVAL);
       
+      testNodeWithoutCollectionForwarding();
+     
       indexr(id, 1, i1, 100, tlong, 100, t1,
           "now is the time for all good men", "foo_f", 1.414f, "foo_b", "true",
           "foo_d", 1.414d);
@@ -154,6 +159,52 @@ public class BasicDistributedZk2Test ext
     
   }
   
+  private void testNodeWithoutCollectionForwarding() throws Exception,
+      SolrServerException, IOException {
+    try {
+      final String baseUrl = ((HttpSolrServer) clients.get(0)).getBaseURL().substring(
+          0,
+          ((HttpSolrServer) clients.get(0)).getBaseURL().length()
+              - DEFAULT_COLLECTION.length() - 1);
+      HttpSolrServer server = new HttpSolrServer(baseUrl);
+      server.setConnectionTimeout(15000);
+      server.setSoTimeout(30000);
+      Create createCmd = new Create();
+      createCmd.setRoles("none");
+      createCmd.setCoreName(ONE_NODE_COLLECTION + "core");
+      createCmd.setCollection(ONE_NODE_COLLECTION);
+      createCmd.setNumShards(1);
+      createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+          + ONE_NODE_COLLECTION);
+      server.request(createCmd);
+    } catch (Exception e) {
+      e.printStackTrace();
+      //fail
+    }
+    
+    waitForRecoveriesToFinish(ONE_NODE_COLLECTION, cloudClient.getZkStateReader(), false);
+    
+    final String baseUrl2 = ((HttpSolrServer) clients.get(1)).getBaseURL().substring(
+        0,
+        ((HttpSolrServer) clients.get(1)).getBaseURL().length()
+            - DEFAULT_COLLECTION.length() - 1);
+    HttpSolrServer qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection" + "core");
+    
+    // add a doc
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", "1");
+    qclient.add(doc);
+    qclient.commit();
+    
+    SolrQuery query = new SolrQuery("*:*");
+    QueryResponse results = qclient.query(query);
+    assertEquals(1, results.getResults().getNumFound());
+    
+    qclient = new HttpSolrServer(baseUrl2 + "/onenodecollection");
+    results = qclient.query(query);
+    assertEquals(1, results.getResults().getNumFound());
+  }
+  
   private long testUpdateAndDelete() throws Exception {
     long docId = 99999999L;
     indexr("id", docId, t1, "originalcontent");



Mime
View raw message