lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1339701 - in /lucene/dev/trunk/solr: cloud-dev/ core/src/java/org/apache/solr/cloud/ core/src/java/org/apache/solr/core/ example/multicore/ example/multicore/core0/conf/ example/multicore/core1/conf/
Date Thu, 17 May 2012 16:47:39 GMT
Author: markrmiller
Date: Thu May 17 16:47:39 2012
New Revision: 1339701

URL: http://svn.apache.org/viewvc?rev=1339701&view=rev
Log:
SOLR-3460: Improve cmd line config bootstrap tool so that it now reads solr.xml and uploads
configs for each SolrCore found

Added:
    lucene/dev/trunk/solr/cloud-dev/solrcloud-multi-start.sh
Modified:
    lucene/dev/trunk/solr/cloud-dev/solrcloud-start.sh
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/CoreContainer.java
    lucene/dev/trunk/solr/example/multicore/   (props changed)
    lucene/dev/trunk/solr/example/multicore/core0/conf/schema.xml
    lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml
    lucene/dev/trunk/solr/example/multicore/core1/conf/schema.xml
    lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml

Added: lucene/dev/trunk/solr/cloud-dev/solrcloud-multi-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/cloud-dev/solrcloud-multi-start.sh?rev=1339701&view=auto
==============================================================================
--- lucene/dev/trunk/solr/cloud-dev/solrcloud-multi-start.sh (added)
+++ lucene/dev/trunk/solr/cloud-dev/solrcloud-multi-start.sh Thu May 17 16:47:39 2012
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# starts up the multicore example
+
+cd ..
+
+rm -r -f example2
+rm -r -f example3
+rm -r -f example4
+rm -r -f example5
+rm -r -f example6
+
+rm -r -f dist
+rm -r -f build
+rm -r -f example/solr/zoo_data
+rm -r -f example/solr/data
+rm -f example/example.log
+
+ant example dist
+
+cp -r -f example example2
+cp -r -f example example3
+cp -r -f example example4
+cp -r -f example example5
+cp -r -f example example6
+
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983
example/multicore 8983
+
+cd example
+java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dsolr.solr.home=multicore -jar
start.jar 1>example.log 2>&1 &
+
+cd ../example2
+java -Djetty.port=7574 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6574 -DSTOP.KEY=key
-Dsolr.solr.home=multicore -jar start.jar 1>example2.log 2>&1 &
+
+cd ../example3
+java -Djetty.port=7575 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6575 -DSTOP.KEY=key
-Dsolr.solr.home=multicore -jar start.jar 1>example3.log 2>&1 &
+
+cd ../example4
+java -Djetty.port=7576 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6576 -DSTOP.KEY=key
-Dsolr.solr.home=multicore -jar start.jar 1>example4.log 2>&1 &
+
+cd ../example5
+java -Djetty.port=7577 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6577 -DSTOP.KEY=key
-Dsolr.solr.home=multicore -jar start.jar 1>example5.log 2>&1 &
+
+cd ../example6
+java -Djetty.port=7578 -DzkHost=localhost:9983 -DnumShards=2 -DSTOP.PORT=6578 -DSTOP.KEY=key
-Dsolr.solr.home=multicore -jar start.jar 1>example6.log 2>&1 &

Modified: lucene/dev/trunk/solr/cloud-dev/solrcloud-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/cloud-dev/solrcloud-start.sh?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/cloud-dev/solrcloud-start.sh (original)
+++ lucene/dev/trunk/solr/cloud-dev/solrcloud-start.sh Thu May 17 16:47:39 2012
@@ -22,7 +22,7 @@ cp -r -f example example4
 cp -r -f example example5
 cp -r -f example example6
 
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983
8983 example/solr/conf conf1 example/solr
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983
example/solr 8983
 
 cd example
 java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log
2>&1 &

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu May 17
16:47:39 2012
@@ -32,6 +32,8 @@ import java.util.concurrent.TimeoutExcep
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.xml.xpath.XPathConstants;
+
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
 import org.apache.solr.common.SolrException;
@@ -46,16 +48,22 @@ import org.apache.solr.common.cloud.ZkNo
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.cloud.ZooKeeperException;
 import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.Config;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.update.UpdateLog;
+import org.apache.solr.util.DOMUtil;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
 
 /**
  * Handle ZooKeeper interactions.
@@ -114,21 +122,31 @@ public final class ZkController {
   // this can be null in which case recovery will be inactive
   private CoreContainer cc;
 
+  /**
+   * Bootstraps the current configs for all collections in solr.xml.
+   * Takes two params - the zkhost to connect to and the solrhome location
+   * to find solr.xml.
+   *
+   * If you also pass a solrPort, it will be used to start
+   * an embedded zk useful for single machine, multi node tests.
+   * 
+   * @param args
+   * @throws Exception
+   */
   public static void main(String[] args) throws Exception {
     // start up a tmp zk server first
     String zkServerAddress = args[0];
     
-    String solrPort = args[1];
-    
-    String confDir = args[2];
-    String confName = args[3];
-    
-    String solrHome = null;
-    if (args.length == 5) {
-      solrHome = args[4];
+    String solrHome = args[1];
+   
+    String solrPort = null;
+    if (args.length > 2) {
+      solrPort = args[2];
     }
+    
+
     SolrZkServer zkServer = null;
-    if (solrHome != null) {
+    if (solrPort != null) {
       zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data", solrHome, solrPort);
       zkServer.parseConfig();
       zkServer.start();
@@ -140,8 +158,13 @@ public final class ZkController {
           public void command() {
           }});
     
-    uploadConfigDir(zkClient, new File(confDir), confName);
-    if (solrHome != null) {
+    SolrResourceLoader loader = new SolrResourceLoader(solrHome);
+    solrHome = loader.getInstanceDir();
+    
+    InputSource cfgis = new InputSource(new File(solrHome, "solr.xml").toURI().toASCIIString());
+    Config cfg = new Config(loader, null, cfgis , null, false);
+    bootstrapConf(zkClient, cfg, solrHome);
+    if (solrPort != null) {
       zkServer.stop();
     }
   }
@@ -855,7 +878,7 @@ public final class ZkController {
         try {
           Map<String,String> collectionProps = new HashMap<String,String>();
           // TODO: if collection.configName isn't set, and there isn't already a conf in
zk, just use that?
-          String defaultConfigName = System.getProperty(COLLECTION_PARAM_PREFIX+CONFIGNAME_PROP,
"configuration1");
+          String defaultConfigName = System.getProperty(COLLECTION_PARAM_PREFIX+CONFIGNAME_PROP,
collection);
 
           // params passed in - currently only done via core admin (create core commmand).
           if (params != null) {
@@ -948,6 +971,13 @@ public final class ZkController {
         collectionProps.put(CONFIGNAME_PROP,  configNames.get(0));
         break;
       }
+      
+      if (configNames != null && configNames.contains(collection)) {
+        log.info("Could not find explicit collection configName, but found config name matching
collection name - using that set.");
+        collectionProps.put(CONFIGNAME_PROP,  collection);
+        break;
+      }
+      
       log.info("Could not find collection configName - pausing for 3 seconds and trying again
- try: " + retry);
       Thread.sleep(3000);
     }
@@ -1155,5 +1185,34 @@ public final class ZkController {
     }
     return leaderProps;
   }
+  
+  /**
+   * If in SolrCloud mode, upload config sets for each SolrCore in solr.xml.
+   * 
+   * @throws IOException
+   * @throws KeeperException
+   * @throws InterruptedException
+   */
+  public static void bootstrapConf(SolrZkClient zkClient, Config cfg, String solrHome) throws
IOException,
+      KeeperException, InterruptedException {
+    
+    NodeList nodes = (NodeList)cfg.evaluate("solr/cores/core", XPathConstants.NODESET);
+
+    for (int i=0; i<nodes.getLength(); i++) {
+      Node node = nodes.item(i);
+      String rawName = DOMUtil.getAttr(node, "name", null);
+      String instanceDir = DOMUtil.getAttr(node, "instanceDir", null);
+      File idir = new File(instanceDir);
+      if (!idir.isAbsolute()) {
+        idir = new File(solrHome, instanceDir);
+      }
+      String confName = DOMUtil.getAttr(node, "collection", null);
+      if (confName == null) {
+        confName = rawName;
+      }
+
+      ZkController.uploadConfigDir(zkClient, new File(idir, "conf"), confName);
+    }
+  }
 
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/CoreContainer.java Thu May 17
16:47:39 2012
@@ -233,7 +233,7 @@ public class CoreContainer 
         
         boolean boostrapConf = Boolean.getBoolean("bootstrap_conf");
         if(boostrapConf) {
-          bootstrapConf();
+          ZkController.bootstrapConf(zkController.getZkClient(), cfg, solrHome);
         }
         
       } catch (InterruptedException e) {
@@ -259,28 +259,6 @@ public class CoreContainer 
     
   }
 
-  private void bootstrapConf() throws IOException,
-      KeeperException, InterruptedException {
-
-    NodeList nodes = (NodeList)cfg.evaluate("solr/cores/core", XPathConstants.NODESET);
-
-    for (int i=0; i<nodes.getLength(); i++) {
-      Node node = nodes.item(i);
-      String rawName = DOMUtil.getAttr(node, "name", null);
-      String instanceDir = DOMUtil.getAttr(node, "instanceDir", null);
-      File idir = new File(instanceDir);
-      if (!idir.isAbsolute()) {
-        idir = new File(solrHome, instanceDir);
-      }
-      String confName = DOMUtil.getAttr(node, "collection", null);
-      if (confName == null) {
-        confName = rawName;
-      }
-
-      zkController.uploadConfigDir(new File(idir, "conf"), confName);
-    }
-  }
-
   public Properties getContainerProperties() {
     return containerProperties;
   }

Modified: lucene/dev/trunk/solr/example/multicore/core0/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/multicore/core0/conf/schema.xml?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/multicore/core0/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/example/multicore/core0/conf/schema.xml Thu May 17 16:47:39 2012
@@ -19,14 +19,16 @@
 <schema name="example core zero" version="1.1">
   <types>
    <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+   <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
   </types>
 
  <fields>   
   <!-- general -->
-  <field name="id"      type="string"   indexed="true"  stored="true"  multiValued="false"
required="true"/>
-  <field name="type"    type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
-  <field name="name"    type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
-  <field name="core0"   type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="id"        type="string"   indexed="true"  stored="true"  multiValued="false"
required="true"/>
+  <field name="type"      type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="name"      type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="core0"     type="string"   indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="_version_" type="long"     indexed="true"  stored="true"/>
  </fields>
 
  <!-- field to use to determine and enforce document uniqueness. -->

Modified: lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/example/multicore/core0/conf/solrconfig.xml Thu May 17 16:47:39
2012
@@ -29,7 +29,22 @@
 
   <dataDir>${solr.core0.data.dir:}</dataDir>
 
-  <updateHandler class="solr.DirectUpdateHandler2" />
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.core0.data.dir:}</str>
+    </updateLog>
+  </updateHandler>
+
+  <!-- realtime get handler, guaranteed to return the latest stored fields 
+    of any document, without the need to commit or open a new searcher. The current 
+    implementation relies on the updateLog feature being enabled. -->
+  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+    <lst name="defaults">
+      <str name="omitHeader">true</str>
+    </lst>
+  </requestHandler>  
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />

 
   <requestDispatcher handleSelect="true" >
     <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />

Modified: lucene/dev/trunk/solr/example/multicore/core1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/multicore/core1/conf/schema.xml?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/multicore/core1/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/example/multicore/core1/conf/schema.xml Thu May 17 16:47:39 2012
@@ -19,14 +19,16 @@
 <schema name="example core one" version="1.1">
   <types>
    <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+   <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
   </types>
 
  <fields>   
   <!-- general -->
-  <field name="id"       type="string"    indexed="true"  stored="true"  multiValued="false"
required="true"/>
-  <field name="type"     type="string"    indexed="true"  stored="true"  multiValued="false"
/> 
-  <field name="name"     type="string"    indexed="true"  stored="true"  multiValued="false"
/> 
-  <field name="core1"    type="string"    indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="id"        type="string"    indexed="true"  stored="true"  multiValued="false"
required="true"/>
+  <field name="type"      type="string"    indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="name"      type="string"    indexed="true"  stored="true"  multiValued="false"
/> 
+  <field name="core1"     type="string"    indexed="true"  stored="true"  multiValued="false"
/>
+  <field name="_version_" type="long"      indexed="true"  stored="true"/>
  </fields>
 
  <!-- field to use to determine and enforce document uniqueness. -->

Modified: lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml?rev=1339701&r1=1339700&r2=1339701&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/example/multicore/core1/conf/solrconfig.xml Thu May 17 16:47:39
2012
@@ -29,7 +29,22 @@
 
   <dataDir>${solr.core1.data.dir:}</dataDir>
 
-  <updateHandler class="solr.DirectUpdateHandler2" />
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.core1.data.dir:}</str>
+    </updateLog>
+  </updateHandler>
+
+  <!-- realtime get handler, guaranteed to return the latest stored fields 
+    of any document, without the need to commit or open a new searcher. The current 
+    implementation relies on the updateLog feature being enabled. -->
+  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+    <lst name="defaults">
+      <str name="omitHeader">true</str>
+    </lst>
+  </requestHandler>
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />

 
   <requestDispatcher handleSelect="true" >
     <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />



Mime
View raw message