lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1360469 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/solr/core/ solr/ solr/cloud-dev/ solr/core/ solr/core/src/java/org/apache/solr/cloud/ solr/core/src/test/org/apache/solr/cloud/ ...
Date Wed, 11 Jul 2012 23:11:33 GMT
Author: markrmiller
Date: Wed Jul 11 23:11:32 2012
New Revision: 1360469

URL: http://svn.apache.org/viewvc?rev=1360469&view=rev
Log:
SOLR-3460: Add cloud-scripts directory and a zkcli.sh|bat tool for easy scripting and interaction
with ZooKeeper.
SOLR-3609: Change Solr's expanded webapp directory to be at a consistent path called solr-webapp
rather than a temporary directory.

Added:
    lucene/dev/branches/branch_4x/solr/cloud-dev/cli-test-solrcloud-start.sh
      - copied unchanged from r1360416, lucene/dev/trunk/solr/cloud-dev/cli-test-solrcloud-start.sh
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
      - copied unchanged from r1360416, lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
      - copied unchanged from r1360416, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
    lucene/dev/branches/branch_4x/solr/example/cloud-scripts/
      - copied from r1360416, lucene/dev/trunk/solr/example/cloud-scripts/
    lucene/dev/branches/branch_4x/solr/example/contexts/
      - copied from r1360416, lucene/dev/trunk/solr/example/contexts/
    lucene/dev/branches/branch_4x/solr/lib/commons-cli-LICENSE-ASL.txt
      - copied unchanged from r1360416, lucene/dev/trunk/solr/lib/commons-cli-LICENSE-ASL.txt
    lucene/dev/branches/branch_4x/solr/lib/commons-cli-NOTICE.txt
      - copied unchanged from r1360416, lucene/dev/trunk/solr/lib/commons-cli-NOTICE.txt
Removed:
    lucene/dev/branches/branch_4x/solr/example/work/
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath
    lucene/dev/branches/branch_4x/dev-tools/maven/pom.xml.template
    lucene/dev/branches/branch_4x/dev-tools/maven/solr/core/pom.xml.template
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/   (props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-multi-start.sh
    lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/ivy.xml
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/example/etc/jetty.xml
    lucene/dev/branches/branch_4x/solr/lib/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java

Modified: lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath (original)
+++ lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath Wed Jul 11 23:11:32 2012
@@ -103,6 +103,7 @@
 	<classpathentry kind="lib" path="lucene/benchmark/lib/commons-compress-1.2.jar"/>
 	<classpathentry kind="lib" path="lucene/benchmark/lib/xercesImpl-2.9.1.jar"/>
 	<classpathentry kind="lib" path="solr/lib/commons-fileupload-1.2.1.jar"/>
+	<classpathentry kind="lib" path="solr/lib/commons-cli-1.2.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpclient-4.1.3.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpcore-4.1.4.jar"/>
 	<classpathentry kind="lib" path="solr/lib/httpmime-4.1.3.jar"/>

Modified: lucene/dev/branches/branch_4x/dev-tools/maven/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/maven/pom.xml.template?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/maven/pom.xml.template (original)
+++ lucene/dev/branches/branch_4x/dev-tools/maven/pom.xml.template Wed Jul 11 23:11:32 2012
@@ -156,6 +156,11 @@
         <version>1.6</version>
       </dependency>
       <dependency>
+        <groupId>commons-cli</groupId>
+        <artifactId>commons-cli</artifactId>
+        <version>1.2</version>
+      </dependency>
+      <dependency>
         <groupId>commons-digester</groupId>
         <artifactId>commons-digester</artifactId>
         <version>1.7</version>

Modified: lucene/dev/branches/branch_4x/dev-tools/maven/solr/core/pom.xml.template
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/maven/solr/core/pom.xml.template?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/maven/solr/core/pom.xml.template (original)
+++ lucene/dev/branches/branch_4x/dev-tools/maven/solr/core/pom.xml.template Wed Jul 11 23:11:32
2012
@@ -139,6 +139,10 @@
       <artifactId>commons-codec</artifactId>
     </dependency>
     <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+    </dependency>
+    <dependency>
       <groupId>commons-fileupload</groupId>
       <artifactId>commons-fileupload</artifactId>
     </dependency>

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Jul 11 23:11:32 2012
@@ -52,6 +52,9 @@ New Features
   
 * SOLR-2616: Include a sample java util logging configuration file. 
   (David Smiley, Mark Miller)
+ 
+* SOLR-3460: Add cloud-scripts directory and a zkcli.sh|bat tool for easy scripting
+  and interaction with ZooKeeper. (Mark Miller)
 
 
 Bug Fixes
@@ -84,6 +87,8 @@ Other Changes
 * SOLR-3599: Add zkClientTimeout to solr.xml so that it's obvious how to change it and so

   that you can change it with a system property. (Mark Miller)
 
+* SOLR-3609: Change Solr's expanded webapp directory to be at a consistent path called
+  solr-webapp rather than a temporary directory. (Mark Miller)
 
 ==================  4.0.0-ALPHA ==================
 More information about this release, including any errata related to the 

Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-multi-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-multi-start.sh?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-multi-start.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-multi-start.sh Wed Jul 11 23:11:32
2012
@@ -24,7 +24,7 @@ cp -r -f example example4
 cp -r -f example example5
 cp -r -f example example6
 
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983
example/multicore 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconf
-zkhost 127.0.0.1:9983 -solrhome example/multicore -runzk 8983
 
 cd example
 java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -Dsolr.solr.home=multicore -jar
start.jar 1>example.log 2>&1 &

Modified: lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh (original)
+++ lucene/dev/branches/branch_4x/solr/cloud-dev/solrcloud-start.sh Wed Jul 11 23:11:32 2012
@@ -22,7 +22,7 @@ cp -r -f example example4
 cp -r -f example example5
 cp -r -f example example6
 
-java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkController 127.0.0.1:9983
example/solr 8983
+java -classpath lib/*:dist/*:build/lucene-libs/* org.apache.solr.cloud.ZkCLI -cmd upconf
-zkhost 127.0.0.1:9983 -solrhome example/solr -runzk 8983
 
 cd example
 java -DzkRun -DnumShards=2 -DSTOP.PORT=7983 -DSTOP.KEY=key -jar start.jar 1>example.log
2>&1 &

Modified: lucene/dev/branches/branch_4x/solr/core/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/ivy.xml?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/ivy.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/ivy.xml Wed Jul 11 23:11:32 2012
@@ -22,6 +22,7 @@
     <dependencies>
       <dependency org="commons-codec" name="commons-codec" rev="1.6" transitive="false"/>
       <dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
+      <dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.1.4" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpclient" rev="4.1.3" transitive="false"/>
       <dependency org="org.apache.httpcomponents" name="httpmime" rev="4.1.3" transitive="false"/>

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
Wed Jul 11 23:11:32 2012
@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
 
 import javax.xml.xpath.XPathConstants;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
 import org.apache.solr.common.SolrException;
@@ -50,8 +51,8 @@ import org.apache.solr.core.Config;
 import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.CoreDescriptor;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.handler.component.HttpShardHandlerFactory;
+
 import org.apache.solr.handler.component.ShardHandler;
 import org.apache.solr.update.UpdateLog;
 import org.apache.solr.util.DOMUtil;
@@ -63,7 +64,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
 
 /**
  * Handle ZooKeeper interactions.
@@ -122,53 +122,6 @@ public final class ZkController {
   private CoreContainer cc;
 
   /**
-   * Bootstraps the current configs for all collections in solr.xml.
-   * Takes two params - the zkhost to connect to and the solrhome location
-   * to find solr.xml.
-   *
-   * If you also pass a solrPort, it will be used to start
-   * an embedded zk useful for single machine, multi node tests.
-   * 
-   * @param args
-   * @throws Exception
-   */
-  public static void main(String[] args) throws Exception {
-    // start up a tmp zk server first
-    String zkServerAddress = args[0];
-    
-    String solrHome = args[1];
-   
-    String solrPort = null;
-    if (args.length > 2) {
-      solrPort = args[2];
-    }
-    
-
-    SolrZkServer zkServer = null;
-    if (solrPort != null) {
-      zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data", solrHome, solrPort);
-      zkServer.parseConfig();
-      zkServer.start();
-    }
-    
-    SolrZkClient zkClient = new SolrZkClient(zkServerAddress, 15000, 5000,
-        new OnReconnect() {
-          @Override
-          public void command() {
-          }});
-    
-    SolrResourceLoader loader = new SolrResourceLoader(solrHome);
-    solrHome = loader.getInstanceDir();
-    
-    InputSource cfgis = new InputSource(new File(solrHome, "solr.xml").toURI().toASCIIString());
-    Config cfg = new Config(loader, null, cfgis , null, false);
-    bootstrapConf(zkClient, cfg, solrHome);
-    if (solrPort != null) {
-      zkServer.stop();
-    }
-  }
-
-  /**
    * @param cc if null, recovery will not be enabled
    * @param zkServerAddress
    * @param zkClientTimeout
@@ -898,9 +851,7 @@ public final class ZkController {
           
           ZkNodeProps zkProps = new ZkNodeProps(collectionProps);
           zkClient.makePath(collectionPath, ZkStateReader.toJSON(zkProps), CreateMode.PERSISTENT,
null, true);
-         
-          // ping that there is a new collection
-          zkClient.setData(ZkStateReader.COLLECTIONS_ZKNODE, (byte[])null, true);
+
         } catch (KeeperException e) {
           // its okay if the node already exists
           if (e.code() != KeeperException.Code.NODEEXISTS) {
@@ -1008,6 +959,24 @@ public final class ZkController {
     }
   }
   
+  public static void downloadFromZK(SolrZkClient zkClient, String zkPath,
+      File dir) throws IOException, KeeperException, InterruptedException {
+    List<String> files = zkClient.getChildren(zkPath, null, true);
+    
+    for (String file : files) {
+      List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
+      if (children.size() == 0) {
+        byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
+        dir.mkdirs(); 
+        log.info("Write file " + new File(dir, file));
+        FileUtils.writeStringToFile(new File(dir, file), new String(data, "UTF-8"), "UTF-8");
+      } else {
+        downloadFromZK(zkClient, zkPath + "/" + file, new File(dir, file));
+      }
+    }
+  }
+  
+  
   private String getCoreNodeName(CoreDescriptor descriptor){
     return getNodeName() + "_"
         + descriptor.getName();
@@ -1016,6 +985,10 @@ public final class ZkController {
   public static void uploadConfigDir(SolrZkClient zkClient, File dir, String configName)
throws IOException, KeeperException, InterruptedException {
     uploadToZK(zkClient, dir, ZkController.CONFIGS_ZKNODE + "/" + configName);
   }
+  
+  public static void downloadConfigDir(SolrZkClient zkClient, String configName, File dir)
throws IOException, KeeperException, InterruptedException {
+    downloadFromZK(zkClient, ZkController.CONFIGS_ZKNODE + "/" + configName, dir);
+  }
 
   public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException
{
     // before becoming available, make sure we are not live and active
@@ -1100,6 +1073,50 @@ public final class ZkController {
     return leaderProps;
   }
   
+  public static void linkConfSet(SolrZkClient zkClient, String collection, String confSetName)
throws KeeperException, InterruptedException {
+    String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection;
+    if (log.isInfoEnabled()) {
+      log.info("Load collection config from:" + path);
+    }
+    byte[] data;
+    try {
+      data = zkClient.getData(path, null, null, true);
+    } catch (NoNodeException e) {
+      // if there is no node, we will try and create it
+      // first try to make in case we are pre configuring
+      ZkNodeProps props = new ZkNodeProps(CONFIGNAME_PROP, confSetName);
+      try {
+
+        zkClient.makePath(path, ZkStateReader.toJSON(props),
+            CreateMode.PERSISTENT, null, true);
+      } catch (KeeperException e2) {
+        // its okay if the node already exists
+        if (e2.code() != KeeperException.Code.NODEEXISTS) {
+          throw e;
+        }
+        // if we fail creating, setdata
+        // TODO: we should consider using version
+        zkClient.setData(path, ZkStateReader.toJSON(props), true);
+      }
+      return;
+    }
+    // we found existing data, let's update it
+    ZkNodeProps props = null;
+    if(data != null) {
+      props = ZkNodeProps.load(data);
+      Map<String,String> newProps = new HashMap<String,String>();
+      newProps.putAll(props.getProperties());
+      newProps.put(CONFIGNAME_PROP, confSetName);
+      props = new ZkNodeProps(newProps);
+    } else {
+      props = new ZkNodeProps(CONFIGNAME_PROP, confSetName);
+    }
+    
+    // TODO: we should consider using version
+    zkClient.setData(path, ZkStateReader.toJSON(props), true);
+
+  }
+  
   /**
    * If in SolrCloud mode, upload config sets for each SolrCore in solr.xml.
    * 
@@ -1114,18 +1131,20 @@ public final class ZkController {
 
     for (int i=0; i<nodes.getLength(); i++) {
       Node node = nodes.item(i);
-      String rawName = DOMUtil.getAttr(node, "name", null);
+      String rawName = DOMUtil.substituteProperty(DOMUtil.getAttr(node, "name", null), new
Properties());
+
       String instanceDir = DOMUtil.getAttr(node, "instanceDir", null);
       File idir = new File(instanceDir);
       if (!idir.isAbsolute()) {
         idir = new File(solrHome, instanceDir);
       }
-      String confName = DOMUtil.getAttr(node, "collection", null);
+      String confName = DOMUtil.substituteProperty(DOMUtil.getAttr(node, "collection", null),
new Properties());
       if (confName == null) {
         confName = rawName;
       }
-
-      ZkController.uploadConfigDir(zkClient, new File(idir, "conf"), confName);
+      File udir = new File(idir, "conf");
+      SolrException.log(log, "Uploading directory " + udir + " with name " + confName + "
for SolrCore " + rawName);
+      ZkController.uploadConfigDir(zkClient, udir, confName);
     }
   }
 

Modified: lucene/dev/branches/branch_4x/solr/example/etc/jetty.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/example/etc/jetty.xml?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/example/etc/jetty.xml (original)
+++ lucene/dev/branches/branch_4x/solr/example/etc/jetty.xml Wed Jul 11 23:11:32 2012
@@ -134,17 +134,14 @@
     </Call>
     
     <Ref id="DeploymentManager">
-          <Call id="webappprovider" name="addAppProvider">
-            <Arg>
-              <New class="org.eclipse.jetty.deploy.providers.WebAppProvider">
-                <Set name="monitoredDirName"><Property name="jetty.home" default="."
/>/webapps</Set>
-                <Set name="defaultsDescriptor"><Property name="jetty.home" default="."/>/etc/webdefault.xml</Set>
-                <Set name="scanInterval">0</Set>
-                <Set name="contextXmlDir"><Property name="jetty.home" default="."
/>/contexts</Set>
-		<Set name="extractWars">true</Set>
-              </New>
-            </Arg>
-          </Call>
+      <Call name="addAppProvider">
+        <Arg>
+          <New class="org.eclipse.jetty.deploy.providers.ContextProvider">
+            <Set name="monitoredDirName"><SystemProperty name="jetty.home" default="."/>/contexts</Set>
+            <Set name="scanInterval">0</Set>
+          </New>
+        </Arg>
+      </Call>
     </Ref>
 
 </Configure>

Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java?rev=1360469&r1=1360468&r2=1360469&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
(original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
Wed Jul 11 23:11:32 2012
@@ -689,10 +689,16 @@ public class SolrZkClient {
       return;
     }
     for (String string : children) {
-      clean(path + "/" + string);
+      if (path.equals("/")) {
+        clean(path + string);
+      } else {
+        clean(path + "/" + string);
+      }
     }
     try {
-      delete(path, -1, true);
+      if (!path.equals("/")) {
+        delete(path, -1, true);
+      }
     } catch (NoNodeException r) {
       return;
     }



Mime
View raw message