lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject svn commit: r1297785 [11/16] - in /lucene/dev/branches/lucene3795_lsp_spatial_module: ./ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/maven/lucene/contrib/demo/ dev-tools/maven/lucene/contrib/highlighter/ dev-tools/maven/lucene...
Date Tue, 06 Mar 2012 23:17:23 GMT
Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java Tue Mar  6 23:17:08 2012
@@ -17,16 +17,6 @@ package org.apache.solr.handler.dataimpo
  * limitations under the License.
  */
 
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
 import org.apache.solr.client.solrj.SolrQuery;
@@ -37,9 +27,20 @@ import org.apache.solr.client.solrj.impl
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CommonParams;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
 /**
  * <p>
  * An implementation of {@link EntityProcessor} which fetches values from a
@@ -58,15 +59,8 @@ public class SolrEntityProcessor extends
   
   public static final String SOLR_SERVER = "url";
   public static final String QUERY = "query";
-  /**
-   * (format="javabin|xml") default is javabin
-   */
-  public static final String FORMAT = "format";
-  public static final String ROWS = "rows";
-  public static final String FIELDS = "fields";
-  public static final String FQ = "fq";
   public static final String TIMEOUT = "timeout";
-  
+
   public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes
   public static final int ROWS_DEFAULT = 50;
   
@@ -75,10 +69,22 @@ public class SolrEntityProcessor extends
   private int rows = ROWS_DEFAULT;
   private String[] filterQueries;
   private String[] fields;
+  private String queryType;
   private int timeout = TIMEOUT_SECS;
   
   private boolean initDone = false;
-  
+
+  /**
+   * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service.
+   * One can override this method to return a differently configured {@link HttpClient} instance.
+   * For example configure https and http authentication.
+   *
+   * @return a {@link HttpClient} instance used for interfacing with a source Solr service
+   */
+  protected HttpClient getHttpClient() {
+    return new HttpClient(new MultiThreadedHttpConnectionManager());
+  }
+
   @Override
   protected void firstInit(Context context) {
     super.firstInit(context);
@@ -89,23 +95,21 @@ public class SolrEntityProcessor extends
         throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
             "SolrEntityProcessor: parameter 'url' is required");
       }
-      HttpClient client = new HttpClient(
-          new MultiThreadedHttpConnectionManager());
+
+      HttpClient client = getHttpClient();
       URL url = new URL(serverPath);
-      
-      if ("xml".equals(context.getResolvedEntityAttribute(FORMAT))) {
-        solrServer = new CommonsHttpSolrServer(url, client,
-            new XMLResponseParser(), false);
+      // (wt="javabin|xml") default is javabin
+      if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) {
+        solrServer = new CommonsHttpSolrServer(url, client, new XMLResponseParser(), false);
         LOG.info("using XMLResponseParser");
       } else {
         solrServer = new CommonsHttpSolrServer(url, client);
         LOG.info("using BinaryResponseParser");
       }
-      
     } catch (MalformedURLException e) {
       throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
     }
-    
+
     this.queryString = context.getResolvedEntityAttribute(QUERY);
     if (this.queryString == null) {
       throw new DataImportHandlerException(
@@ -114,21 +118,21 @@ public class SolrEntityProcessor extends
       );
     }
     
-    String rowsP = context.getResolvedEntityAttribute(ROWS);
+    String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS);
     if (rowsP != null) {
       rows = Integer.parseInt(rowsP);
     }
     
-    String fqAsString = context.getResolvedEntityAttribute(FQ);
+    String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ);
     if (fqAsString != null) {
       this.filterQueries = fqAsString.split(",");
     }
     
-    String fieldsAsString = context.getResolvedEntityAttribute(FIELDS);
+    String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL);
     if (fieldsAsString != null) {
       this.fields = fieldsAsString.split(",");
     }
-    
+    this.queryType = context.getResolvedEntityAttribute(CommonParams.QT);
     String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
     if (timeoutAsString != null) {
       this.timeout = Integer.parseInt(timeoutAsString);
@@ -181,6 +185,7 @@ public class SolrEntityProcessor extends
         solrQuery.addField(field);
       }
     }
+    solrQuery.setQueryType(queryType);
     solrQuery.setFilterQueries(filterQueries);
     solrQuery.setTimeAllowed(timeout * 1000);
     

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestScriptTransformer.java Tue Mar  6 23:17:08 2012
@@ -53,7 +53,8 @@ public class TestScriptTransformer exten
       sep.applyTransformer(map);
       assertEquals(map.get("name"), "Hello Scott");
     } catch (DataImportHandlerException e) {
-      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", "<script> can be used only in java 6 or above".equals(e.getMessage()));
+      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", 
+                  e.getMessage().startsWith("<script> can be used only in java 6 or above"));      
       throw e;
     }
   }
@@ -85,7 +86,8 @@ public class TestScriptTransformer exten
       sep.applyTransformer(map);
       assertEquals(map.get("name"), "Hello Scott");
     } catch (DataImportHandlerException e) {
-      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", "<script> can be used only in java 6 or above".equals(e.getMessage()));
+      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", 
+                  e.getMessage().startsWith("<script> can be used only in java 6 or above"));      
       throw e;
     }
   }
@@ -101,7 +103,8 @@ public class TestScriptTransformer exten
               .item(0));
       assertTrue(config.script.text.indexOf("checkNextToken") > -1);
     } catch (DataImportHandlerException e) {
-      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", "<script> can be used only in java 6 or above".equals(e.getMessage()));
+      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", 
+                  e.getMessage().startsWith("<script> can be used only in java 6 or above"));      
       throw e;
     }
   }
@@ -129,7 +132,8 @@ public class TestScriptTransformer exten
       sep.applyTransformer(map);
       assertNull(map.get("$hasMore"));
     } catch (DataImportHandlerException e) {
-      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", "<script> can be used only in java 6 or above".equals(e.getMessage()));
+      assumeFalse("JRE does not contain a JavaScript engine (OpenJDK)", 
+                  e.getMessage().startsWith("<script> can be used only in java 6 or above"));      
       throw e;
     }
   }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java Tue Mar  6 23:17:08 2012
@@ -145,7 +145,7 @@ public class TestSolrEntityProcessorEndT
     
     try {
       addDocumentsToSolr(SOLR_DOCS);
-      runFullImport(generateDIHConfig("query='*:*' rows='2' fields='id,desc' onError='skip'", jetty.getLocalPort()));
+      runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort()));
     } catch (Exception e) {
       LOG.error(e.getMessage(), e);
       fail(e.getMessage());
@@ -178,7 +178,7 @@ public class TestSolrEntityProcessorEndT
     
     try {
       addDocumentsToSolr(generateSolrDocuments(7));
-      runFullImport(generateDIHConfig("query='*:*' fields='id' rows='2'", jetty.getLocalPort()));
+      runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'", jetty.getLocalPort()));
     } catch (Exception e) {
       LOG.error(e.getMessage(), e);
       fail(e.getMessage());
@@ -231,7 +231,7 @@ public class TestSolrEntityProcessorEndT
     assertQ(req("*:*"), "//result[@numFound='0']");
     
     try {
-      runFullImport(generateDIHConfig("query='*:*' rows='2' fields='id,desc' onError='skip'", jetty.getLocalPort()));
+      runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'", jetty.getLocalPort()));
     } catch (Exception e) {
       LOG.error(e.getMessage(), e);
       fail(e.getMessage());
@@ -244,7 +244,7 @@ public class TestSolrEntityProcessorEndT
     assertQ(req("*:*"), "//result[@numFound='0']");
     
     try {
-      runFullImport(generateDIHConfig("query='bogus:3' rows='2' fields='id,desc' onError='abort'", jetty.getLocalPort()));
+      runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='abort'", jetty.getLocalPort()));
     } catch (Exception e) {
       LOG.error(e.getMessage(), e);
       fail(e.getMessage());

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java Tue Mar  6 23:17:08 2012
@@ -211,7 +211,7 @@ public class TestXPathEntityProcessor ex
     tmpdir.delete();
     tmpdir.mkdir();
     tmpdir.deleteOnExit();
-    TestFileListEntityProcessor.createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"),
+    AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"),
             false);
     Map entityAttrs = createMap("name", "e",
             XPathEntityProcessor.USE_SOLR_ADD_SCHEMA, "true", "xsl", ""

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/dataimport.jsp Tue Mar  6 23:17:08 2012
@@ -47,7 +47,7 @@ Select handler:
 
 <frameset cols = "50%, 50%">
   <frame src ="debug.jsp?handler=<%=handler%>" />
-  <frame src ="../select?qt=<%=handler%>&command=status"  name="result"/>
+  <frame src ="..<%=handler%>?command=status"  name="result"/>
 </frameset>
 <% } %>
 </html>

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/dataimporthandler/src/webapp/admin/debug.jsp Tue Mar  6 23:17:08 2012
@@ -30,9 +30,8 @@
 <%
   String handler = request.getParameter("handler");  // must be specified
 %>
-<form action="../select" target="result" method="get">
+<form action="..<%=handler%>" target="result" method="get">
 <input type="hidden" name="debug" value="on"/>
-<input type="hidden" name="qt" value="<%=handler%>"/>
 <table>
   <tr>
     <th>Handler: </th>
@@ -69,17 +68,16 @@
 	<tr>
 		<td colspan="2"><textarea id="txtDataConfig" rows="30" cols="80" name="dataConfig"></textarea></td>
     <script type="text/javascript" language="javascript">
-        $.get("../select?qt=<%=handler%>&command=show-config", function(data){
+        $.get("..<%=handler%>?command=show-config", function(data){
             $('#txtDataConfig').attr('value', data);
         });
     </script>
 	</tr>
 </table>
 </form>
-<form action="../select" target="result" method="get">
+<form action="..<%=handler%>" target="result" method="get">
 	<input type="hidden" name="clean" value="false">
 	<input type="hidden" name="commit" value="true">
-  <input type="hidden" name="qt" value="<%=handler%>"/>
 	<input class="stdbutton" type="submit" name="command" value="full-import">
 	<input class="stdbutton" type="submit" name="command" value="delta-import">
 	<input class="stdbutton" type="submit" name="command" value="status">
@@ -92,8 +90,7 @@
 	<input type="hidden" name="rows" value="0">
 	<input class="stdbutton" type="submit" value="Documents Count">
 </form>
-<form action="../select" target="result" method="get">
-  <input type="hidden" name="qt" value="<%=handler%>"/>
+<form action="..<%=handler%>" target="result" method="get">
 	<input type="hidden" name="verbose" value="true">
 	<input type="hidden" name="clean" value="true">
 	<input type="hidden" name="commit" value="true">

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java Tue Mar  6 23:17:08 2012
@@ -36,7 +36,9 @@ import org.apache.solr.update.AddUpdateC
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.tika.config.TikaConfig;
 import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.HttpHeaders;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.DefaultParser;
@@ -150,11 +152,11 @@ public class ExtractingDocumentLoader ex
       // then Tika can make use of it in guessing the appropriate MIME type:
       String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null);
       if (resourceName != null) {
-        metadata.add(Metadata.RESOURCE_NAME_KEY, resourceName);
+        metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);
       }
       // Provide stream's content type as hint for auto detection
       if(stream.getContentType() != null) {
-        metadata.add(Metadata.CONTENT_TYPE, stream.getContentType());
+        metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());
       }
 
       InputStream inputStream = null;
@@ -167,7 +169,7 @@ public class ExtractingDocumentLoader ex
         // HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata
         String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
         if(charset != null){
-          metadata.add(Metadata.CONTENT_ENCODING, charset);
+          metadata.add(HttpHeaders.CONTENT_ENCODING, charset);
         }
 
         String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java Tue Mar  6 23:17:08 2012
@@ -24,6 +24,7 @@ import org.apache.solr.schema.DateField;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaMetadataKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.Attributes;
@@ -191,7 +192,7 @@ public class SolrContentHandler extends 
     if (sf==null && unknownFieldPrefix.length() > 0) {
       name = unknownFieldPrefix + name;
       sf = schema.getFieldOrNull(name);
-    } else if (sf == null && defaultField.length() > 0 && name.equals(Metadata.RESOURCE_NAME_KEY) == false /*let the fall through below handle this*/){
+    } else if (sf == null && defaultField.length() > 0 && name.equals(TikaMetadataKeys.RESOURCE_NAME_KEY) == false /*let the fall through below handle this*/){
       name = defaultField;
       sf = schema.getFieldOrNull(name);
     }
@@ -201,7 +202,7 @@ public class SolrContentHandler extends 
     // ExtractingDocumentLoader.load(). You shouldn't have to define a mapping for this
     // field just because you specified a resource.name parameter to the handler, should
     // you?
-    if (sf == null && unknownFieldPrefix.length()==0 && name == Metadata.RESOURCE_NAME_KEY) {
+    if (sf == null && unknownFieldPrefix.length()==0 && name == TikaMetadataKeys.RESOURCE_NAME_KEY) {
       return;
     }
 

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/langid/src/test-files/langid/solr/conf/solrconfig-languageidentifier.xml Tue Mar  6 23:17:08 2012
@@ -31,7 +31,7 @@
         solr.RAMDirectoryFactory is memory based and not persistent. -->
   <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
 
-  <luceneMatchVersion>LUCENE_40</luceneMatchVersion>
+  <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
 
   <requestHandler name="search" class="solr.SearchHandler" default="true">
      <lst name="defaults">

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/CHANGES.txt?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/CHANGES.txt (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/CHANGES.txt Tue Mar  6 23:17:08 2012
@@ -5,9 +5,12 @@ This file describes changes to the Solr 
 
 Introduction
 ------------
-This module is intended to be used while indexing documents.
-Its purpose is to provide additional on the fly automatically generated fields to the Solr index.
+This module is intended to be used both as an UpdateRequestProcessor while indexing documents and as a set of tokenizer/filters
+to be configured inside the schema.xml for use during analysis phase.
+UIMAUpdateRequestProcessor purpose is to provide additional on the fly automatically generated fields to the Solr index.
 Such fields could be language, concepts, keywords, sentences, named entities, etc.
+UIMA based tokenizers/filters can be used either inside plain Lucene or as index/query analyzers to be defined
+inside the schema.xml of a Solr core to create/filter tokens using specific UIMA annotations.
 
  UIMA Dependency
  ---------------

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/build.xml?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/build.xml (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/build.xml Tue Mar  6 23:17:08 2012
@@ -20,9 +20,25 @@
 <project name="solr-uima" default="default">
 
   <description>
-    Solr Integration with UIMA for extracting metadata from arbitrary (text) fields and enrich document with features extracted from UIMA types (language, sentences, concepts, named entities, etc.)
+    Solr Integration with UIMA for extracting metadata from arbitrary (text) fields and enrich document with features
+    extracted from UIMA types (language, sentences, concepts, named entities, etc.)
   </description>
 
   <import file="../contrib-build.xml"/>
+  
+  <path id="classpath">
+    <pathelement path="${analyzers-uima.jar}"/>
+    <path refid="solr.base.classpath"/>
+  </path>
+
+  <target name="module-jars-to-solr" depends="jar-analyzers-uima">
+    <mkdir dir="${build.dir}/lucene-libs"/>
+    <copy todir="${build.dir}/lucene-libs" preservelastmodified="true" flatten="true" failonerror="true" overwrite="true">
+      <fileset file="${analyzers-uima.jar}"/>
+    </copy>
+  </target>
+
+  <target name="compile-core" depends="jar-analyzers-uima, solr-contrib-build.compile-core"/>
+  <target name="dist" depends="module-jars-to-solr, common-solr.dist"/>
 
 </project>

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/uima/src/java/org/apache/solr/uima/processor/UIMAUpdateRequestProcessor.java Tue Mar  6 23:17:08 2012
@@ -23,14 +23,16 @@ import org.apache.solr.common.SolrInputD
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.uima.processor.SolrUIMAConfiguration.MapField;
-import org.apache.solr.uima.processor.ae.AEProvider;
-import org.apache.solr.uima.processor.ae.AEProviderFactory;
+import org.apache.lucene.analysis.uima.ae.AEProvider;
+import org.apache.lucene.analysis.uima.ae.AEProviderFactory;
 import org.apache.solr.update.AddUpdateCommand;
 import org.apache.solr.update.processor.UpdateRequestProcessor;
 import org.apache.uima.analysis_engine.AnalysisEngine;
 import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
 import org.apache.uima.jcas.JCas;
 import org.apache.uima.resource.ResourceInitializationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Map;
@@ -42,6 +44,8 @@ import java.util.Map;
  */
 public class UIMAUpdateRequestProcessor extends UpdateRequestProcessor {
 
+  private final Logger log = LoggerFactory.getLogger(UIMAUpdateRequestProcessor.class);
+
   SolrUIMAConfiguration solrUIMAConfiguration;
 
   private AEProvider aeProvider;

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java Tue Mar  6 23:17:08 2012
@@ -25,6 +25,7 @@ import org.apache.solr.request.SolrQuery
 import org.apache.velocity.Template;
 import org.apache.velocity.VelocityContext;
 import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.runtime.RuntimeConstants;
 import org.apache.velocity.tools.generic.*;
 
 import java.io.*;
@@ -117,14 +118,14 @@ public class VelocityResponseWriter impl
     if (template_root != null) {
       baseDir = new File(template_root);
     }
-    engine.setProperty(VelocityEngine.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath());
+    engine.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, baseDir.getAbsolutePath());
     engine.setProperty("params.resource.loader.instance", new SolrParamResourceLoader(request));
     SolrVelocityResourceLoader resourceLoader =
         new SolrVelocityResourceLoader(request.getCore().getSolrConfig().getResourceLoader());
     engine.setProperty("solr.resource.loader.instance", resourceLoader);
 
     // TODO: Externalize Velocity properties
-    engine.setProperty(VelocityEngine.RESOURCE_LOADER, "params,file,solr");
+    engine.setProperty(RuntimeConstants.RESOURCE_LOADER, "params,file,solr");
     String propFile = request.getParams().get("v.properties");
     try {
       if (propFile == null)

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/FSTSynonymFilterFactory.java Tue Mar  6 23:17:08 2012
@@ -157,6 +157,9 @@ final class FSTSynonymFilterFactory exte
   private static TokenizerFactory loadTokenizerFactory(ResourceLoader loader, String cname, Map<String,String> args){
     TokenizerFactory tokFactory = (TokenizerFactory) loader.newInstance(cname);
     tokFactory.init(args);
+    if (tokFactory instanceof ResourceLoaderAware) {
+      ((ResourceLoaderAware) tokFactory).inform(loader);
+    }
     return tokFactory;
   }
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java Tue Mar  6 23:17:08 2012
@@ -28,8 +28,7 @@ import java.util.Map;
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer;
-import org.apache.lucene.analysis.kuromoji.Segmenter;
-import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
+import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Mode;
 import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
 import org.apache.lucene.util.IOUtils;
 import org.apache.solr.analysis.BaseTokenizerFactory;
@@ -88,7 +87,7 @@ public class KuromojiTokenizerFactory ex
   
   @Override
   public Tokenizer create(Reader input) {
-    return new KuromojiTokenizer(new Segmenter(userDictionary, mode), input);
+    return new KuromojiTokenizer(input, userDictionary, true, mode);
   }
   
   private Mode getMode(Map<String, String> args) {
@@ -96,7 +95,7 @@ public class KuromojiTokenizerFactory ex
     if (mode != null) {
       return Mode.valueOf(mode.toUpperCase(Locale.ENGLISH));
     } else {
-      return Segmenter.DEFAULT_MODE;
+      return KuromojiTokenizer.DEFAULT_MODE;
     }
   }
 }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/analysis/SlowSynonymFilterFactory.java Tue Mar  6 23:17:08 2012
@@ -171,6 +171,9 @@ final class SlowSynonymFilterFactory ext
   private static TokenizerFactory loadTokenizerFactory(ResourceLoader loader, String cname, Map<String,String> args){
     TokenizerFactory tokFactory = (TokenizerFactory)loader.newInstance( cname );
     tokFactory.init( args );
+    if (tokFactory instanceof ResourceLoaderAware) {
+      ((ResourceLoaderAware) tokFactory).inform(loader);
+    }
     return tokFactory;
   }
 

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java Tue Mar  6 23:17:08 2012
@@ -155,6 +155,9 @@ public class EmbeddedSolrServer extends 
       
       core.execute( handler, req, rsp );
       if( rsp.getException() != null ) {
+        if(rsp.getException() instanceof SolrException) {
+          throw rsp.getException();
+        }
         throw new SolrServerException( rsp.getException() );
       }
       
@@ -219,6 +222,9 @@ public class EmbeddedSolrServer extends 
     catch( IOException iox ) {
       throw iox;
     }
+    catch( SolrException sx ) {
+      throw sx;
+    }
     catch( Exception ex ) {
       throw new SolrServerException( ex );
     }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Tue Mar  6 23:17:08 2012
@@ -8,6 +8,7 @@ import org.apache.solr.common.SolrExcept
 import org.apache.solr.common.cloud.CloudState;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkClientConnectionStrategy;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
@@ -40,19 +41,25 @@ public abstract class ElectionContext {
   final ZkNodeProps leaderProps;
   final String id;
   final String leaderPath;
+  String leaderSeqPath;
+  private SolrZkClient zkClient;
   
   public ElectionContext(final String shardZkNodeName,
-      final String electionPath, final String leaderPath, final ZkNodeProps leaderProps) {
+      final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
     this.id = shardZkNodeName;
     this.electionPath = electionPath;
     this.leaderPath = leaderPath;
     this.leaderProps = leaderProps;
+    this.zkClient = zkClient;
   }
   
+  public void cancelElection() throws InterruptedException, KeeperException {
+    zkClient.delete(leaderSeqPath, -1, true);
+  }
   // the given core may or may not be null - if you need access to the current core, you must pass
   // the core container and core name to your context impl - then use this core ref if it is not null
   // else access it from the core container
-  abstract void runLeaderProcess(String leaderSeqPath, boolean weAreReplacement, SolrCore core) throws KeeperException, InterruptedException, IOException;
+  abstract void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException, IOException;
 }
 
 class ShardLeaderElectionContextBase extends ElectionContext {
@@ -66,7 +73,7 @@ class ShardLeaderElectionContextBase ext
       final String collection, final String shardZkNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
     super(shardZkNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
         + shardId, ZkStateReader.getShardLeadersPath(collection, shardId),
-        props);
+        props, zkStateReader.getZkClient());
     this.leaderElector = leaderElector;
     this.zkClient = zkStateReader.getZkClient();
     this.shardId = shardId;
@@ -74,7 +81,7 @@ class ShardLeaderElectionContextBase ext
   }
 
   @Override
-  void runLeaderProcess(String leaderSeqPath, boolean weAreReplacement, SolrCore core)
+  void runLeaderProcess(boolean weAreReplacement)
       throws KeeperException, InterruptedException, IOException {
 
     try {
@@ -109,7 +116,7 @@ final class ShardLeaderElectionContext e
   }
   
   @Override
-  void runLeaderProcess(String leaderSeqPath, boolean weAreReplacement, SolrCore startupCore)
+  void runLeaderProcess(boolean weAreReplacement)
       throws KeeperException, InterruptedException, IOException {
     if (cc != null) {
       String coreName = leaderProps.get(ZkStateReader.CORE_NAME_PROP);
@@ -117,13 +124,12 @@ final class ShardLeaderElectionContext e
       try {
         // the first time we are run, we will get a startupCore - after
         // we will get null and must use cc.getCore
-        if (startupCore == null) {
-          core = cc.getCore(coreName);
-        } else {
-          core = startupCore;
-        }
+     
+        core = cc.getCore(coreName);
+
         if (core == null) {
-          throw new SolrException(ErrorCode.SERVER_ERROR, "Core not found:" + coreName);
+          cancelElection();
+          throw new SolrException(ErrorCode.SERVER_ERROR, "Fatal Error, SolrCore not found:" + coreName + " in " + cc.getCoreNames());
         }
         // should I be leader?
         if (weAreReplacement && !shouldIBeLeader(leaderProps)) {
@@ -131,7 +137,7 @@ final class ShardLeaderElectionContext e
           rejoinLeaderElection(leaderSeqPath, core);
           return;
         }
-        
+
         if (weAreReplacement) {
           if (zkClient.exists(leaderPath, true)) {
             zkClient.delete(leaderPath, -1, true);
@@ -139,44 +145,41 @@ final class ShardLeaderElectionContext e
 //          System.out.println("I may be the new Leader:" + leaderPath
 //              + " - I need to try and sync");
           boolean success = syncStrategy.sync(zkController, core, leaderProps);
-          if (!success) {
-            // TODO: what if no one can be the leader in a loop?
-            // perhaps we look down the list and if no one is active, we
-            // accept leader role anyhow
-            core.getUpdateHandler().getSolrCoreState().doRecovery(core);
-            
+          if (!success && anyoneElseActive()) {
             rejoinLeaderElection(leaderSeqPath, core);
             return;
           } 
         }
         
         // If I am going to be the leader I have to be active
-        
+        // System.out.println("I am leader go active");
         core.getUpdateHandler().getSolrCoreState().cancelRecovery();
         zkController.publish(core.getCoreDescriptor(), ZkStateReader.ACTIVE);
         
       } finally {
-        if (core != null && startupCore == null) {
+        if (core != null ) {
           core.close();
         }
       }
       
     }
     
-    super.runLeaderProcess(leaderSeqPath, weAreReplacement, startupCore);
+    super.runLeaderProcess(weAreReplacement);
   }
 
   private void rejoinLeaderElection(String leaderSeqPath, SolrCore core)
       throws InterruptedException, KeeperException, IOException {
     // remove our ephemeral and re join the election
-   // System.out.println("sync failed, delete our election node:"
-   //     + leaderSeqPath);
+    // System.out.println("sync failed, delete our election node:"
+    // + leaderSeqPath);
+
     zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN);
-    zkClient.delete(leaderSeqPath, -1, true);
     
-    core.getUpdateHandler().getSolrCoreState().doRecovery(core);
+    cancelElection();
+    
+    core.getUpdateHandler().getSolrCoreState().doRecovery(cc, core.getName());
     
-    leaderElector.joinElection(this, null);
+    leaderElector.joinElection(this);
   }
   
   private boolean shouldIBeLeader(ZkNodeProps leaderProps) {
@@ -210,6 +213,26 @@ final class ShardLeaderElectionContext e
     return !foundSomeoneElseActive;
   }
   
+  private boolean anyoneElseActive() {
+    CloudState cloudState = zkController.getZkStateReader().getCloudState();
+    Map<String,Slice> slices = cloudState.getSlices(this.collection);
+    Slice slice = slices.get(shardId);
+    Map<String,ZkNodeProps> shards = slice.getShards();
+
+    for (Map.Entry<String,ZkNodeProps> shard : shards.entrySet()) {
+      String state = shard.getValue().get(ZkStateReader.STATE_PROP);
+
+      
+      if ((state.equals(ZkStateReader.ACTIVE))
+          && cloudState.liveNodesContain(shard.getValue().get(
+              ZkStateReader.NODE_NAME_PROP))) {
+        return true;
+      }
+    }
+    
+    return false;
+  }
+  
 }
 
 final class OverseerElectionContext extends ElectionContext {
@@ -218,13 +241,13 @@ final class OverseerElectionContext exte
   private final ZkStateReader stateReader;
 
   public OverseerElectionContext(final String zkNodeName, SolrZkClient zkClient, ZkStateReader stateReader) {
-    super(zkNodeName, "/overseer_elect", "/overseer_elect/leader", null);
+    super(zkNodeName, "/overseer_elect", "/overseer_elect/leader", null, stateReader.getZkClient());
     this.zkClient = zkClient;
     this.stateReader = stateReader;
   }
 
   @Override
-  void runLeaderProcess(String leaderSeqPath, boolean weAreReplacement, SolrCore firstCore) throws KeeperException, InterruptedException {
+  void runLeaderProcess(boolean weAreReplacement) throws KeeperException, InterruptedException {
     
     final String id = leaderSeqPath.substring(leaderSeqPath.lastIndexOf("/")+1);
     ZkNodeProps myProps = new ZkNodeProps("id", id);

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Tue Mar  6 23:17:08 2012
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
  * Leader Election process. This class contains the logic by which a
  * leader is chosen. First call * {@link #setup(ElectionContext)} to ensure
  * the election process is init'd. Next call
- * {@link #joinElection(ElectionContext, SolrCore)} to start the leader election.
+ * {@link #joinElection(ElectionContext)} to start the leader election.
  * 
  * The implementation follows the classic ZooKeeper recipe of creating an
  * ephemeral, sequential node for each candidate and then looking at the set
@@ -80,13 +80,12 @@ public  class LeaderElector {
    * @param seq
    * @param context 
    * @param replacement has someone else been the leader already?
-   * @param core 
    * @throws KeeperException
    * @throws InterruptedException
    * @throws IOException 
    * @throws UnsupportedEncodingException
    */
-  private void checkIfIamLeader(final String leaderSeqPath, final int seq, final ElectionContext context, boolean replacement, SolrCore core) throws KeeperException,
+  private void checkIfIamLeader(final int seq, final ElectionContext context, boolean replacement) throws KeeperException,
       InterruptedException, IOException {
     // get all other numbers...
     final String holdElectionPath = context.electionPath + ELECTION_NODE;
@@ -95,7 +94,7 @@ public  class LeaderElector {
     sortSeqs(seqs);
     List<Integer> intSeqs = getSeqs(seqs);
     if (seq <= intSeqs.get(0)) {
-      runIamLeaderProcess(leaderSeqPath, context, replacement, core);
+      runIamLeaderProcess(context, replacement);
     } else {
       // I am not the leader - watch the node below me
       int i = 1;
@@ -119,7 +118,7 @@ public  class LeaderElector {
               public void process(WatchedEvent event) {
                 // am I the next leader?
                 try {
-                  checkIfIamLeader(leaderSeqPath, seq, context, true, null);
+                  checkIfIamLeader(seq, context, true);
                 } catch (InterruptedException e) {
                   // Restore the interrupted status
                   Thread.currentThread().interrupt();
@@ -137,16 +136,15 @@ public  class LeaderElector {
       } catch (KeeperException e) {
         // we couldn't set our watch - the node before us may already be down?
         // we need to check if we are the leader again
-        checkIfIamLeader(leaderSeqPath, seq, context, true, null);
+        checkIfIamLeader(seq, context, true);
       }
     }
   }
 
   // TODO: get this core param out of here
-  protected void runIamLeaderProcess(String leaderSeqPath, final ElectionContext context, boolean weAreReplacement, SolrCore core) throws KeeperException,
+  protected void runIamLeaderProcess(final ElectionContext context, boolean weAreReplacement) throws KeeperException,
       InterruptedException, IOException {
-
-    context.runLeaderProcess(leaderSeqPath, weAreReplacement, core);
+    context.runLeaderProcess(weAreReplacement);
   }
   
   /**
@@ -207,7 +205,7 @@ public  class LeaderElector {
    * @throws IOException 
    * @throws UnsupportedEncodingException
    */
-  public int joinElection(ElectionContext context, SolrCore core) throws KeeperException, InterruptedException, IOException {
+  public int joinElection(ElectionContext context) throws KeeperException, InterruptedException, IOException {
     final String shardsElectZkPath = context.electionPath + LeaderElector.ELECTION_NODE;
     
     long sessionId = zkClient.getSolrZooKeeper().getSessionId();
@@ -219,6 +217,7 @@ public  class LeaderElector {
       try {
         leaderSeqPath = zkClient.create(shardsElectZkPath + "/" + id + "-n_", null,
             CreateMode.EPHEMERAL_SEQUENTIAL, false);
+        context.leaderSeqPath = leaderSeqPath;
         cont = false;
       } catch (ConnectionLossException e) {
         // we don't know if we made our node or not...
@@ -249,7 +248,7 @@ public  class LeaderElector {
       }
     }
     int seq = getSeq(leaderSeqPath);
-    checkIfIamLeader(leaderSeqPath, seq, context, false, core);
+    checkIfIamLeader(seq, context, false);
     
     return seq;
   }

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/NodeStateWatcher.java Tue Mar  6 23:17:08 2012
@@ -18,6 +18,7 @@ package org.apache.solr.cloud;
  */
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -42,6 +43,8 @@ public class NodeStateWatcher implements
   public static interface NodeStateChangeListener {
     void coreChanged(String nodeName, Set<CoreState> states)
         throws KeeperException, InterruptedException;
+    void coreDeleted(String nodeName, Collection<CoreState> states)
+    throws KeeperException, InterruptedException;
   }
 
   private final SolrZkClient zkClient;
@@ -104,6 +107,19 @@ public class NodeStateWatcher implements
           }
         }
         
+        HashMap<String, CoreState> deletedCores = new HashMap<String, CoreState>();
+        for(CoreState state: currentState) {
+          deletedCores.put(state.getCoreNodeName(), state);
+        }
+
+        for(CoreState state: stateList) {
+          deletedCores.remove(state.getCoreNodeName());
+        }
+
+        if (deletedCores.size() > 0) {
+          listener.coreDeleted(nodeName, deletedCores.values());
+        }
+        
         currentState = Collections.unmodifiableSet(newState);
 
         if (modifiedCores.size() > 0) {

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue Mar  6 23:17:08 2012
@@ -57,7 +57,7 @@ public class Overseer implements NodeSta
   private static final int STATE_UPDATE_DELAY = 500;  // delay between cloud state updates
 
   static enum Op {
-    LeaderChange, StateChange; 
+    LeaderChange, StateChange, CoreDeleted; 
   }
 
   private final class CloudStateUpdateRequest {
@@ -135,6 +135,9 @@ public class Overseer implements NodeSta
                         (String) request.args[0], (CoreState) request.args[1]);
                     break;
 
+                  case CoreDeleted:
+                    cloudState = removeCore(cloudState, (String) request.args[0], (String) request.args[1]);
+                    break;
                   }
                 }
 
@@ -168,7 +171,7 @@ public class Overseer implements NodeSta
       
       private boolean amILeader() {
         try {
-          ZkNodeProps props = ZkNodeProps.load(zkClient.getData("/overseer_elect/leader", null, null, false));
+          ZkNodeProps props = ZkNodeProps.load(zkClient.getData("/overseer_elect/leader", null, null, true));
           if(myId.equals(props.get("id"))) {
             return true;
           }
@@ -294,7 +297,6 @@ public class Overseer implements NodeSta
       
       private CloudState setShardLeader(CloudState state, String collection, String sliceName, String leaderUrl) {
         
-        boolean updated = false;
         final Map<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>();
         newStates.putAll(state.getCollectionStates());
         
@@ -314,32 +316,49 @@ public class Overseer implements NodeSta
             Map<String, String> newShardProps = new LinkedHashMap<String,String>();
             newShardProps.putAll(shard.getValue().getProperties());
             
-            String wasLeader = newShardProps.remove(ZkStateReader.LEADER_PROP);  //clean any previously existed flag
-
+            newShardProps.remove(ZkStateReader.LEADER_PROP);  //clean any previously existed flag
+            
             ZkCoreNodeProps zkCoreNodeProps = new ZkCoreNodeProps(new ZkNodeProps(newShardProps));
             if(leaderUrl!=null && leaderUrl.equals(zkCoreNodeProps.getCoreUrl())) {
               newShardProps.put(ZkStateReader.LEADER_PROP,"true");
-              if (wasLeader == null) {
-                updated = true;
-              }
-            } else {
-              if (wasLeader != null) {
-                updated = true;
-              }
             }
             newShards.put(shard.getKey(), new ZkNodeProps(newShardProps));
           }
           Slice slice = new Slice(sliceName, newShards);
           slices.put(sliceName, slice);
         }
-        if (updated) {
-          return new CloudState(state.getLiveNodes(), newStates);
-        } else {
-          return state;
-        }
+        return new CloudState(state.getLiveNodes(), newStates);
       }
-
-    }
+      
+      /*
+       * Remove core from cloudstate
+       */
+      private CloudState removeCore(final CloudState cloudState, final String collection, final String coreNodeName) {
+        final LinkedHashMap<String, Map<String, Slice>> newStates = new LinkedHashMap<String,Map<String,Slice>>();
+        for(String collectionName: cloudState.getCollections()) {
+          if(collection.equals(collectionName)) {
+            Map<String, Slice> slices = cloudState.getSlices(collection);
+            LinkedHashMap<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
+            for(Slice slice: slices.values()) {
+              if(slice.getShards().containsKey(coreNodeName)) {
+                LinkedHashMap<String, ZkNodeProps> newShards = new LinkedHashMap<String, ZkNodeProps>();
+                newShards.putAll(slice.getShards());
+                newShards.remove(coreNodeName);
+                Slice newSlice = new Slice(slice.getName(), newShards);
+                newSlices.put(slice.getName(), newSlice);
+              } else {
+                newSlices.put(slice.getName(), slice);
+              }
+            }
+            newStates.put(collectionName, newSlices);
+          } else {
+            newStates.put(collectionName, cloudState.getSlices(collectionName));
+          }
+        }
+        CloudState newState = new CloudState(cloudState.getLiveNodes(), newStates);
+        return newState;
+     }
+  }
   
   public Overseer(final SolrZkClient zkClient, final ZkStateReader reader, String id) throws KeeperException, InterruptedException {
     log.info("Constructing new Overseer id=" + id);
@@ -462,7 +481,6 @@ public class Overseer implements NodeSta
       ShardLeaderWatcher watcher = watches.remove(shardId);
       if (watcher != null) {
         watcher.close();
-        announceLeader(collection, shardId, new ZkCoreNodeProps(new ZkNodeProps()));  //removes loeader for shard
       }
     }
     
@@ -494,9 +512,9 @@ public class Overseer implements NodeSta
                 try {
                     List<String> liveNodes = zkClient.getChildren(
                         ZkStateReader.LIVE_NODES_ZKNODE, this, true);
-                    Set<String> liveNodesSet = new HashSet<String>();
-                    liveNodesSet.addAll(liveNodes);
-                    processLiveNodesChanged(nodeStateWatches.keySet(), liveNodes);
+                    synchronized (nodeStateWatches) {
+                      processLiveNodesChanged(nodeStateWatches.keySet(), liveNodes);
+                    }
                 } catch (KeeperException e) {
                   if (e.code() == KeeperException.Code.SESSIONEXPIRED
                       || e.code() == KeeperException.Code.CONNECTIONLOSS) {
@@ -528,7 +546,9 @@ public class Overseer implements NodeSta
     
     Set<String> downNodes = complement(oldLiveNodes, liveNodes);
     for(String node: downNodes) {
-      NodeStateWatcher watcher = nodeStateWatches.remove(node);
+      synchronized (nodeStateWatches) {
+        NodeStateWatcher watcher = nodeStateWatches.remove(node);
+      }
       log.debug("Removed NodeStateWatcher for node:" + node);
     }
   }
@@ -565,7 +585,15 @@ public class Overseer implements NodeSta
       fifo.add(new CloudStateUpdateRequest(Op.StateChange, nodeName, state));
     }
   }
-  
+
+  @Override
+  public void coreDeleted(String nodeName, Collection<CoreState> states)
+      throws KeeperException, InterruptedException {
+    for (CoreState state : states) {
+      fifo.add(new CloudStateUpdateRequest(Op.CoreDeleted, state.getCollectionName(), state.getCoreNodeName()));
+    }
+  }
+
   public static void createClientNodes(SolrZkClient zkClient, String nodeName) throws KeeperException, InterruptedException {
     final String node = STATES_NODE + "/" + nodeName;
     if (log.isInfoEnabled()) {
@@ -583,4 +611,4 @@ public class Overseer implements NodeSta
     fifo.add(new CloudStateUpdateRequest(Op.LeaderChange, collection, shardId, coreUrl));
   }
   
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java Tue Mar  6 23:17:08 2012
@@ -37,6 +37,7 @@ import org.apache.solr.common.cloud.ZkCo
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.RequestHandlers.LazyRequestHandlerWrapper;
 import org.apache.solr.core.SolrCore;
@@ -44,6 +45,8 @@ import org.apache.solr.handler.Replicati
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.update.CommitUpdateCommand;
 import org.apache.solr.update.PeerSync;
 import org.apache.solr.update.UpdateLog;
@@ -69,14 +72,14 @@ public class RecoveryStrategy extends Th
   private ZkStateReader zkStateReader;
   private volatile String coreName;
   private int retries;
-  private SolrCore core;
   private boolean recoveringAfterStartup;
+  private CoreContainer cc;
   
-  public RecoveryStrategy(SolrCore core) {
-    this.core = core;
-    this.coreName = core.getName();
+  public RecoveryStrategy(CoreContainer cc, String name) {
+    this.cc = cc;
+    this.coreName = name;
     setName("RecoveryThread");
-    zkController = core.getCoreDescriptor().getCoreContainer().getZkController();
+    zkController = cc.getZkController();
     zkStateReader = zkController.getZkStateReader();
     baseUrl = zkController.getBaseUrl();
     coreZkNodeName = zkController.getNodeName() + "_" + coreName;
@@ -103,9 +106,6 @@ public class RecoveryStrategy extends Th
   
   private void replicate(String nodeName, SolrCore core, ZkNodeProps leaderprops, String baseUrl)
       throws SolrServerException, IOException {
-    // start buffer updates to tran log
-    // and do recovery - either replay via realtime get (eventually)
-    // or full index replication
    
     String leaderBaseUrl = leaderprops.get(ZkStateReader.BASE_URL_PROP);
     ZkCoreNodeProps leaderCNodeProps = new ZkCoreNodeProps(leaderprops);
@@ -183,18 +183,42 @@ public class RecoveryStrategy extends Th
     prepCmd.setCoreNodeName(coreZkNodeName);
     prepCmd.setState(ZkStateReader.RECOVERING);
     prepCmd.setCheckLive(true);
-    prepCmd.setPauseFor(4000);
+    prepCmd.setPauseFor(6000);
     
     server.request(prepCmd);
     server.shutdown();
   }
-  
+
   @Override
   public void run() {
+    SolrCore core = cc.getCore(coreName);
+    if (core == null) {
+      SolrException.log(log, "SolrCore not found - cannot recover:" + coreName);
+      return;
+    }
+
+    // set request info for logging
+    try {
+      SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
+      SolrQueryResponse rsp = new SolrQueryResponse();
+      SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+
+      log.info("Starting recovery process. recoveringAfterStartup=" + recoveringAfterStartup);
+
+      doRecovery(core);
+    } finally {
+      if (core != null) core.close();
+      SolrRequestInfo.clearRequestInfo();
+    }
+  }
+
+  // TODO: perhaps make this grab a new core each time through the loop to handle core reloads?
+  public void doRecovery(SolrCore core) {
     boolean replayed = false;
-    boolean succesfulRecovery = false;
+    boolean successfulRecovery = false;
 
-    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
+    UpdateLog ulog;
+    ulog = core.getUpdateHandler().getUpdateLog();
     if (ulog == null) {
       SolrException.log(log, "No UpdateLog found - cannot recover");
       recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
@@ -202,6 +226,7 @@ public class RecoveryStrategy extends Th
       return;
     }
 
+
     List<Long> startingRecentVersions;
     UpdateLog.RecentUpdates startingRecentUpdates = ulog.getRecentUpdates();
     try {
@@ -221,14 +246,14 @@ public class RecoveryStrategy extends Th
         if (startingRecentVersions.get(oldIdx) == firstStartingVersion) break;
       }
 
-      if (oldIdx < startingRecentVersions.size()) {
-        log.info("####### Found new versions added after startup: num=" + (startingRecentVersions.size()-oldIdx));
+      if (oldIdx > 0) {
+        log.info("####### Found new versions added after startup: num=" + oldIdx);
+        log.info("###### currentVersions=" + startingRecentVersions);
       }
-      
+
       log.info("###### startupVersions=" + reallyStartingVersions);
-      log.info("###### currentVersions=" + startingRecentVersions);
     }
-    
+
     if (recoveringAfterStartup) {
       // if we're recovering after startup (i.e. we have been down), then we need to know what the last versions were
       // when we went down.
@@ -237,9 +262,9 @@ public class RecoveryStrategy extends Th
 
     boolean firstTime = true;
 
-    while (!succesfulRecovery && !close && !isInterrupted()) { // don't use interruption or it will close channels though
+    while (!successfulRecovery && !close && !isInterrupted()) { // don't use interruption or it will close channels though
       try {
-
+        // first thing we just try to sync
         zkController.publish(core.getCoreDescriptor(), ZkStateReader.RECOVERING);
 
         CloudDescriptor cloudDesc = core.getCoreDescriptor()
@@ -257,8 +282,10 @@ public class RecoveryStrategy extends Th
 
         // first thing we just try to sync
         if (firstTime) {
-          firstTime = false;    // only try sync the first time through the loop
+          firstTime = false; // only try sync the first time through the loop
           log.info("Attempting to PeerSync from " + leaderUrl + " recoveringAfterStartup="+recoveringAfterStartup);
+          // System.out.println("Attempting to PeerSync from " + leaderUrl
+          // + " i am:" + zkController.getNodeName());
           PeerSync peerSync = new PeerSync(core,
               Collections.singletonList(leaderUrl), ulog.numRecordsToKeep);
           peerSync.setStartingVersions(startingRecentVersions);
@@ -267,42 +294,63 @@ public class RecoveryStrategy extends Th
             SolrQueryRequest req = new LocalSolrQueryRequest(core,
                 new ModifiableSolrParams());
             core.getUpdateHandler().commit(new CommitUpdateCommand(req, false));
-            log.info("Sync Recovery was succesful - registering as Active");
+            log.info("Sync Recovery was successful - registering as Active");
+            // System.out
+            // .println("Sync Recovery was successful - registering as Active "
+            // + zkController.getNodeName());
+
+            // solrcloud_debug
+            // try {
+            // RefCounted<SolrIndexSearcher> searchHolder =
+            // core.getNewestSearcher(false);
+            // SolrIndexSearcher searcher = searchHolder.get();
+            // try {
+            // System.out.println(core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName()
+            // + " synched "
+            // + searcher.search(new MatchAllDocsQuery(), 1).totalHits);
+            // } finally {
+            // searchHolder.decref();
+            // }
+            // } catch (Exception e) {
+            //
+            // }
+
             // sync success - register as active and return
             zkController.publishAsActive(baseUrl, core.getCoreDescriptor(),
                 coreZkNodeName, coreName);
-            succesfulRecovery = true;
+            successfulRecovery = true;
             close = true;
             return;
           }
 
           log.info("Sync Recovery was not successful - trying replication");
         }
+        //System.out.println("Sync Recovery was not successful - trying replication");
 
         log.info("Begin buffering updates");
         ulog.bufferUpdates();
         replayed = false;
-        
+
         try {
-          
+
           replicate(zkController.getNodeName(), core,
               leaderprops, leaderUrl);
-          
+
           replay(ulog);
           replayed = true;
-          
-          log.info("Recovery was succesful - registering as Active");
+
+          log.info("Recovery was successful - registering as Active");
           // if there are pending recovery requests, don't advert as active
           zkController.publishAsActive(baseUrl, core.getCoreDescriptor(),
               coreZkNodeName, coreName);
           close = true;
-          succesfulRecovery = true;
+          successfulRecovery = true;
         } catch (InterruptedException e) {
           Thread.currentThread().interrupt();
           log.warn("Recovery was interrupted", e);
           retries = INTERRUPTED;
         } catch (Throwable t) {
-          SolrException.log(log, "Error while trying to recover", t);
+          log.error("Error while trying to recover", t);
         } finally {
           if (!replayed) {
             try {
@@ -311,36 +359,36 @@ public class RecoveryStrategy extends Th
               SolrException.log(log, "", t);
             }
           }
-          
+
         }
-        
+
       } catch (Throwable t) {
-        SolrException.log(log, "Error while trying to recover", t);
+        log.error("Error while trying to recover.", t);
       }
-      
-      if (!succesfulRecovery) {
+
+      if (!successfulRecovery) {
         // lets pause for a moment and we need to try again...
         // TODO: we don't want to retry for some problems?
         // Or do a fall off retry...
         try {
-          
-          SolrException.log(log, "Recovery failed - trying again...");
+
+          log.error("Recovery failed - trying again...");
           retries++;
           if (retries >= MAX_RETRIES) {
             if (retries == INTERRUPTED) {
-              
+
             } else {
-              // TODO: for now, give up after X tries - should we do more?
+              log.error("Recovery failed - max retries exceeded.");
               recoveryFailed(core, zkController, baseUrl, coreZkNodeName,
                   core.getCoreDescriptor());
             }
             break;
           }
-          
+
         } catch (Exception e) {
           SolrException.log(log, "", e);
         }
-        
+
         try {
           Thread.sleep(Math.min(START_TIMEOUT * retries, 60000));
         } catch (InterruptedException e) {
@@ -349,10 +397,10 @@ public class RecoveryStrategy extends Th
           retries = INTERRUPTED;
         }
       }
-      
-      log.info("Finished recovery process");
-      
+
     }
+    log.info("Finished recovery process");
+
   }
 
   private Future<RecoveryInfo> replay(UpdateLog ulog)

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java Tue Mar  6 23:17:08 2012
@@ -45,17 +45,21 @@ public class SolrZkServer {
   
   String zkRun;
   String zkHost;
-  String solrHome;
+
   String solrPort;
   Properties props;
   SolrZkServerProps zkProps;
 
   private Thread zkThread;  // the thread running a zookeeper server, only if zkRun is set
 
-  public SolrZkServer(String zkRun, String zkHost, String solrHome, String solrPort) {
+  private String dataHome;
+  private String confHome;
+
+  public SolrZkServer(String zkRun, String zkHost, String dataHome, String confHome, String solrPort) {
     this.zkRun = zkRun;
     this.zkHost = zkHost;
-    this.solrHome = solrHome;
+    this.dataHome = dataHome;
+    this.confHome = confHome;
     this.solrPort = solrPort;
   }
 
@@ -74,13 +78,13 @@ public class SolrZkServer {
       zkProps = new SolrZkServerProps();
       // set default data dir
       // TODO: use something based on IP+port???  support ensemble all from same solr home?
-      zkProps.setDataDir(solrHome + '/' + "zoo_data");
+      zkProps.setDataDir(dataHome);
       zkProps.zkRun = zkRun;
       zkProps.solrPort = solrPort;
     }
     
     try {
-      props = SolrZkServerProps.getProperties(solrHome + '/' + "zoo.cfg");
+      props = SolrZkServerProps.getProperties(confHome + '/' + "zoo.cfg");
       SolrZkServerProps.injectServers(props, zkRun, zkHost);
       zkProps.parseProperties(props);
       if (zkProps.getClientPortAddress() == null) {

Modified: lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java?rev=1297785&r1=1297784&r2=1297785&view=diff
==============================================================================
--- lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java (original)
+++ lucene/dev/branches/lucene3795_lsp_spatial_module/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java Tue Mar  6 23:17:08 2012
@@ -23,10 +23,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.params.HttpMethodParams;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery;
-import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.CloudState;
 import org.apache.solr.common.cloud.Slice;
@@ -37,12 +40,42 @@ import org.apache.solr.common.params.Cor
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.component.HttpShardHandlerFactory;
+import org.apache.solr.handler.component.ShardHandler;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
 import org.apache.solr.update.PeerSync;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class SyncStrategy {
   protected final Logger log = LoggerFactory.getLogger(getClass());
+
+  private HttpShardHandlerFactory shardHandlerFactory;
+
+  private ShardHandler shardHandler;
+  
+  private static MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
+  private static HttpClient client = new HttpClient(mgr);
+  static {
+    mgr.getParams().setDefaultMaxConnectionsPerHost(20);
+    mgr.getParams().setMaxTotalConnections(10000);
+    mgr.getParams().setConnectionTimeout(30000);
+    mgr.getParams().setSoTimeout(30000);
+
+    // prevent retries  (note: this didn't work when set on mgr.. needed to be set on client)
+    DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false);
+    client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler);
+  }
+  
+  public SyncStrategy() {
+    shardHandlerFactory = new HttpShardHandlerFactory();
+    shardHandler = shardHandlerFactory.getShardHandler(client);
+  }
+  
+  private static class SyncShardRequest extends ShardRequest {
+    String coreName;
+  }
   
   public boolean sync(ZkController zkController, SolrCore core,
       ZkNodeProps leaderProps) {
@@ -51,6 +84,10 @@ public class SyncStrategy {
     
     // solrcloud_debug
     // System.out.println("SYNC UP");
+    if (core.getUpdateHandler().getUpdateLog() == null) {
+      log.error("No UpdateLog found - cannot sync");
+      return false;
+    }
     boolean success = syncReplicas(zkController, core, leaderProps);
     return success;
   }
@@ -156,7 +193,7 @@ public class SyncStrategy {
     }
     
  
-    PeerSync peerSync = new PeerSync(core, syncWith, 1000);
+    PeerSync peerSync = new PeerSync(core, syncWith, core.getUpdateHandler().getUpdateLog().numRecordsToKeep);
     return peerSync.sync();
   }
   
@@ -180,44 +217,68 @@ public class SyncStrategy {
     ZkCoreNodeProps zkLeader = new ZkCoreNodeProps(leaderProps);
     for (ZkCoreNodeProps node : nodes) {
       try {
-        // TODO: do we first everyone register as sync phase? get the overseer
-        // to do it?
-        // TODO: this should be done in parallel
-        QueryRequest qr = new QueryRequest(params("qt", "/get", "getVersions",
-            Integer.toString(1000), "sync", zkLeader.getCoreUrl(), "distrib",
-            "false"));
-        CommonsHttpSolrServer server = new CommonsHttpSolrServer(
-            node.getCoreUrl());
-        server.setConnectionTimeout(15000);
-        server.setSoTimeout(15000);
-        //System.out.println("ask " + node.getCoreUrl() + " to sync");
-        NamedList rsp = server.request(qr);
-        //System.out.println("response about syncing to leader:" + rsp + " node:"
-        //    + node.getCoreUrl() + " me:" + zkController.getBaseUrl());
-        boolean success = (Boolean) rsp.get("sync");
-        //System.out.println("success:" + success);
-        if (!success) {
-         // System.out
-         //     .println("try and ask " + node.getCoreUrl() + " to recover");
-          log.info("try and ask " + node.getCoreUrl() + " to recover");
-          try {
-            server = new CommonsHttpSolrServer(node.getBaseUrl());
-            server.setSoTimeout(5000);
-            server.setConnectionTimeout(5000);
-            
-            RequestRecovery recoverRequestCmd = new RequestRecovery();
-            recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
-            recoverRequestCmd.setCoreName(node.getCoreName());
-            
-            server.request(recoverRequestCmd);
-          } catch (Exception e) {
-            log.info("Could not tell a replica to recover", e);
-          }
-        }
+//         System.out
+//             .println("try and ask " + node.getCoreUrl() + " to sync");
+        log.info("try and ask " + node.getCoreUrl() + " to sync");
+        requestSync(zkLeader.getCoreUrl(), node.getCoreName());
+
       } catch (Exception e) {
         SolrException.log(log, "Error syncing replica to leader", e);
       }
     }
+    
+    
+    for(;;) {
+      ShardResponse srsp = shardHandler.takeCompletedOrError();
+      if (srsp == null) break;
+      boolean success = handleResponse(srsp);
+      //System.out.println("got response:" + success);
+      if (!success) {
+         try {
+           log.info("Sync failed - asking replica to recover.");
+           //System.out.println("Sync failed - asking replica to recover.");
+           RequestRecovery recoverRequestCmd = new RequestRecovery();
+           recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
+           recoverRequestCmd.setCoreName(((SyncShardRequest)srsp.getShardRequest()).coreName);
+           
+           CommonsHttpSolrServer server = new CommonsHttpSolrServer(zkLeader.getBaseUrl());
+           server.request(recoverRequestCmd);
+         } catch (Exception e) {
+           log.info("Could not tell a replica to recover", e);
+         }
+         shardHandler.cancelAll();
+        break;
+      }
+    }
+  }
+  
+  private boolean handleResponse(ShardResponse srsp) {
+    NamedList<Object> response = srsp.getSolrResponse().getResponse();
+    // TODO: why does this return null sometimes?
+    if (response == null) {
+      return false;
+    }
+    boolean success = (Boolean) response.get("sync");
+    
+    return success;
+  }
+
+  private void requestSync(String replica, String coreName) {
+    SyncShardRequest sreq = new SyncShardRequest();
+    sreq.coreName = coreName;
+    sreq.purpose = 1;
+    // TODO: this sucks
+    if (replica.startsWith("http://"))
+      replica = replica.substring(7);
+    sreq.shards = new String[]{replica};
+    sreq.actualShards = sreq.shards;
+    sreq.params = new ModifiableSolrParams();
+    sreq.params.set("qt","/get");
+    sreq.params.set("distrib",false);
+    sreq.params.set("getVersions",Integer.toString(100));
+    sreq.params.set("sync",replica);
+    
+    shardHandler.submit(sreq, replica, sreq.params);
   }
   
   public static ModifiableSolrParams params(String... params) {



Mime
View raw message