lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1420992 [4/7] - in /lucene/dev/branches/branch_4x: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/codecs/ lucene/core/ lucene/core/src/test/org/ap...
Date Wed, 12 Dec 2012 21:41:26 GMT
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Wed Dec 12 21:41:06 2012
@@ -20,6 +20,7 @@ package org.apache.solr.update.processor
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -37,14 +38,15 @@ import org.apache.solr.common.SolrExcept
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
-import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.cloud.ZooKeeperException;
 import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
 import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.params.UpdateParams;
 import org.apache.solr.common.util.Hash;
@@ -175,30 +177,50 @@ public class DistributedUpdateProcessor 
     }
   }
 
-  private List<Node> setupRequest(int hash) {
+
+  private List<Node> setupRequest(String id, SolrInputDocument doc) {
     List<Node> nodes = null;
 
     // if we are in zk mode...
     if (zkEnabled) {
-      // set num nodes
-      numNodes = zkController.getClusterState().getLiveNodes().size();
-      
-      String shardId = getShard(hash, collection, zkController.getClusterState()); // get the right shard based on the hash...
+
+      String coreName = req.getCore().getName();
+      String coreNodeName = zkController.getNodeName() + "_" + coreName;
+
+      ClusterState cstate = zkController.getClusterState();
+      numNodes = cstate.getLiveNodes().size();
+      DocCollection coll = cstate.getCollection(collection);
+      Slice slice = coll.getRouter().getTargetSlice(id, doc, req.getParams(), coll);
+
+      if (slice == null) {
+        // No slice found.  Most strict routers will have already thrown an exception, so a null return is
+        // a signal to use the slice of this core.
+        // TODO: what if this core is not in the targeted collection?
+        String shardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
+        slice = coll.getSlice(shardId);
+        if (slice == null) {
+          throw new SolrException(ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
+        }
+      }
+
+
+      String shardId = slice.getName();
 
       try {
-        ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderProps(
+        // Not equivalent to getLeaderProps, which does retries to find a leader.
+        // Replica leader = slice.getLeader();
+
+        ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderRetry(
             collection, shardId));
-        
+
         String leaderNodeName = leaderProps.getCoreNodeName();
-        String coreName = req.getCore().getName();
-        String coreNodeName = zkController.getNodeName() + "_" + coreName;
         isLeader = coreNodeName.equals(leaderNodeName);
-        
-        DistribPhase phase = 
+
+        DistribPhase phase =
             DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
-       
+
         doDefensiveChecks(shardId, phase);
-     
+
 
         if (DistribPhase.FROMLEADER == phase) {
           // we are coming from the leader, just go local - add no urls
@@ -219,7 +241,7 @@ public class DistributedUpdateProcessor 
               skipListSet = new HashSet<String>(skipList.length);
               skipListSet.addAll(Arrays.asList(skipList));
             }
-            
+
             for (ZkCoreNodeProps props : replicaProps) {
               if (skipList != null) {
                 if (!skipListSet.contains(props.getCoreUrl())) {
@@ -230,14 +252,14 @@ public class DistributedUpdateProcessor 
               }
             }
           }
-          
+
         } else {
           // I need to forward onto the leader...
           nodes = new ArrayList<Node>(1);
           nodes.add(new RetryNode(leaderProps, zkController.getZkStateReader(), collection, shardId));
           forwardToLeader = true;
         }
-        
+
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();
         throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
@@ -248,6 +270,7 @@ public class DistributedUpdateProcessor 
     return nodes;
   }
 
+
   private void doDefensiveChecks(String shardId, DistribPhase phase) {
     String from = req.getParams().get("distrib.from");
     boolean logReplay = req.getParams().getBool(LOG_REPLAY, false);
@@ -264,13 +287,6 @@ public class DistributedUpdateProcessor 
   }
 
 
-  private String getShard(int hash, String collection, ClusterState clusterState) {
-    // ranges should be part of the cloud state and eventually gotten from zk
-
-    // get the shard names
-    return clusterState.getShard(hash, collection);
-  }
-
   // used for deleteByQuery to get the list of nodes this leader should forward to
   private List<Node> setupRequest() {
     List<Node> nodes = null;
@@ -278,7 +294,7 @@ public class DistributedUpdateProcessor 
 
     try {
 
-      ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderProps(
+      ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderRetry(
           collection, shardId));
 
       String leaderNodeName = leaderProps.getCoreNodeName();
@@ -310,12 +326,9 @@ public class DistributedUpdateProcessor 
 
   @Override
   public void processAdd(AddUpdateCommand cmd) throws IOException {
-    // TODO: check for id field?
-    int hash = 0;
     if (zkEnabled) {
       zkCheck();
-      hash = hash(cmd);
-      nodes = setupRequest(hash);
+      nodes = setupRequest(cmd.getHashableId(), cmd.getSolrInputDocument());
     } else {
       isLeader = getNonZkLeaderAssumption(req);
     }
@@ -684,11 +697,9 @@ public class DistributedUpdateProcessor 
       return;
     }
 
-    int hash = 0;
     if (zkEnabled) {
       zkCheck();
-      hash = hash(cmd);
-      nodes = setupRequest(hash);
+      nodes = setupRequest(cmd.getId(), null);
     } else {
       isLeader = getNonZkLeaderAssumption(req);
     }
@@ -767,22 +778,19 @@ public class DistributedUpdateProcessor 
     if (zkEnabled && DistribPhase.NONE == phase) {
       boolean leaderForAnyShard = false;  // start off by assuming we are not a leader for any shard
 
-      Map<String,Slice> slices = zkController.getClusterState().getSlices(collection);
-      if (slices == null) {
-        throw new SolrException(ErrorCode.BAD_REQUEST,
-            "Cannot find collection:" + collection + " in "
-                + zkController.getClusterState().getCollections());
-      }
+      ModifiableSolrParams outParams = new ModifiableSolrParams(filterParams(req.getParams()));
+      outParams.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
 
-      ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
-      params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
+      DocCollection coll = zkController.getClusterState().getCollection(collection);
+      SolrParams params = req.getParams();
+      Collection<Slice> slices = coll.getRouter().getSearchSlices(params.get(ShardParams.SHARD_KEYS), params, coll);
 
       List<Node> leaders =  new ArrayList<Node>(slices.size());
-      for (Map.Entry<String,Slice> sliceEntry : slices.entrySet()) {
-        String sliceName = sliceEntry.getKey();
-        ZkNodeProps leaderProps;
+      for (Slice slice : slices) {
+        String sliceName = slice.getName();
+        Replica leader;
         try {
-          leaderProps = zkController.getZkStateReader().getLeaderProps(collection, sliceName);
+          leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName);
         } catch (InterruptedException e) {
           throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e);
         }
@@ -791,7 +799,7 @@ public class DistributedUpdateProcessor 
         // should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we get an error because it went down) then look up the new leader?
 
         // Am I the leader for this slice?
-        ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leaderProps);
+        ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leader);
         String leaderNodeName = coreLeaderProps.getCoreNodeName();
         String coreName = req.getCore().getName();
         String coreNodeName = zkController.getNodeName() + "_" + coreName;
@@ -805,8 +813,8 @@ public class DistributedUpdateProcessor 
         }
       }
 
-      params.remove("commit"); // this will be distributed from the local commit
-      cmdDistrib.distribDelete(cmd, leaders, params);
+      outParams.remove("commit"); // this will be distributed from the local commit
+      cmdDistrib.distribDelete(cmd, leaders, outParams);
 
       if (!leaderForAnyShard) {
         return;
@@ -1074,7 +1082,7 @@ public class DistributedUpdateProcessor 
     ClusterState clusterState = req.getCore().getCoreDescriptor()
         .getCoreContainer().getZkController().getClusterState();
     List<Node> urls = new ArrayList<Node>();
-    Map<String,Slice> slices = clusterState.getSlices(collection);
+    Map<String,Slice> slices = clusterState.getSlicesMap(collection);
     if (slices == null) {
       throw new ZooKeeperException(ErrorCode.BAD_REQUEST,
           "Could not find collection in zk: " + clusterState);
@@ -1097,19 +1105,6 @@ public class DistributedUpdateProcessor 
     return urls;
   }
   
-  // TODO: move this to AddUpdateCommand/DeleteUpdateCommand and cache it? And
-  // make the hash pluggable of course.
-  // The hash also needs to be pluggable
-  private int hash(AddUpdateCommand cmd) {
-    String hashableId = cmd.getHashableId();
-    
-    return Hash.murmurhash3_x86_32(hashableId, 0, hashableId.length(), 0);
-  }
-  
-  private int hash(DeleteUpdateCommand cmd) {
-    return Hash.murmurhash3_x86_32(cmd.getId(), 0, cmd.getId().length(), 0);
-  }
-  
   // RetryNodes are used in the case of 'forward to leader' where we want
   // to try the latest leader on a fail in the case the leader just went down.
   public static class RetryNode extends StdNode {
@@ -1134,7 +1129,7 @@ public class DistributedUpdateProcessor 
     public boolean checkRetry() {
       ZkCoreNodeProps leaderProps;
       try {
-        leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderProps(
+        leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
             collection, shardId));
       } catch (InterruptedException e) {
         Thread.currentThread().interrupt();

Added: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesInputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesInputStream.java?rev=1420992&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesInputStream.java (added)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesInputStream.java Wed Dec 12 21:41:06 2012
@@ -0,0 +1,51 @@
+package org.apache.solr.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.lucene.store.IndexInput;
+
+public class PropertiesInputStream extends InputStream {
+  
+  private IndexInput is;
+  
+  public PropertiesInputStream(IndexInput is) {
+    this.is = is;
+  }
+  
+  @Override
+  public int read() throws IOException {
+    byte next;
+    try {
+      next = is.readByte();
+    } catch (EOFException e) {
+      return -1;
+    }
+    return next;
+  }
+  
+  @Override
+  public void close() throws IOException {
+    super.close();
+    is.close();
+  }
+  
+}

Added: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesOutputStream.java?rev=1420992&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesOutputStream.java (added)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/util/PropertiesOutputStream.java Wed Dec 12 21:41:06 2012
@@ -0,0 +1,44 @@
+package org.apache.solr.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.lucene.store.IndexOutput;
+
+public class PropertiesOutputStream extends OutputStream {
+  
+  private IndexOutput out;
+
+  public PropertiesOutputStream(IndexOutput out) {
+    this.out = out;
+  }
+  
+  @Override
+  public void write(int b) throws IOException {
+    out.writeByte((byte) b);
+  }
+  
+  @Override
+  public void close() throws IOException {
+    super.close();
+    out.close();
+  }
+  
+}

Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-luceneMatchVersion.xml Wed Dec 12 21:41:06 2012
@@ -18,12 +18,12 @@
 <schema name="luceneMatchVersionTest" version="1.1">
  <types>
   <fieldtype name="string" class="solr.StrField"/>
-  <fieldtype name="text30" class="solr.TextField">
+  <fieldtype name="text40" class="solr.TextField">
     <analyzer>
-      <tokenizer class="solr.StandardTokenizerFactory" luceneMatchVersion="LUCENE_30"/>
+      <tokenizer class="solr.StandardTokenizerFactory" luceneMatchVersion="LUCENE_40"/>
       <filter class="solr.StandardFilterFactory"/>
       <filter class="solr.LowerCaseFilterFactory"/>
-      <filter class="solr.StopFilterFactory" luceneMatchVersion="3.1"/>
+      <filter class="solr.StopFilterFactory" luceneMatchVersion="4.0"/>
       <filter class="solr.PorterStemFilterFactory"/>
     </analyzer>
   </fieldtype>
@@ -36,8 +36,8 @@
       <filter class="solr.PorterStemFilterFactory"/>
     </analyzer>
   </fieldtype>
-  <fieldtype name="textStandardAnalyzer30" class="solr.TextField">
-    <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer" luceneMatchVersion="LUCENE_30"/>
+  <fieldtype name="textStandardAnalyzer40" class="solr.TextField">
+    <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer" luceneMatchVersion="LUCENE_40"/>
   </fieldtype>
   <fieldtype name="textStandardAnalyzerDefault" class="solr.TextField">
     <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
@@ -46,9 +46,9 @@
  <fields>
    <field name="id" type="string" indexed="true" stored="true"/>
    <field name="signatureField" type="string" indexed="true" stored="false"/>
-   <field name="text30" type="text30" indexed="true" stored="false" />
+   <field name="text40" type="text40" indexed="true" stored="false" />
    <field name="textDefault" type="textDefault" indexed="true" stored="false" />
-   <field name="textStandardAnalyzer30" type="textStandardAnalyzer30" indexed="true" stored="false" />
+   <field name="textStandardAnalyzer40" type="textStandardAnalyzer40" indexed="true" stored="false" />
    <field name="textStandardAnalyzerDefault" type="textStandardAnalyzerDefault" indexed="true" stored="false" />
    <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
  </fields>

Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/solr.xml Wed Dec 12 21:41:06 2012
@@ -29,7 +29,7 @@
     If 'null' (or absent), cores will not be manageable via request handler
   -->
   <cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}" 
-         hostContext="solr" zkClientTimeout="8000" numShards="${numShards:3}" shareSchema="${shareSchema:false}">
+         hostContext="${hostContext:solr}" zkClientTimeout="8000" numShards="${numShards:3}" shareSchema="${shareSchema:false}">
     <core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"/>
   </cores>
 </solr>

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/AnalysisAfterCoreReloadTest.java Wed Dec 12 21:41:06 2012
@@ -23,49 +23,26 @@ import java.io.IOException;
 import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
-import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest.ACTION;
 import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.request.AbstractUpdateRequest.ACTION;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.util.AbstractSolrTestCase;
-import org.junit.After;
+import org.junit.BeforeClass;
 
-public class AnalysisAfterCoreReloadTest extends AbstractSolrTestCase {
-  private File homeDir;
+public class AnalysisAfterCoreReloadTest extends SolrTestCaseJ4 {
+  
   int port = 0;
   static final String context = "/solr";
-  JettySolrRunner jetty;
+
   static final String collection = "collection1";
   
-  @After
-  public void cleanUp() throws Exception {
-    jetty.stop();
-    if (homeDir != null && homeDir.isDirectory() && homeDir.exists())
-      recurseDelete(homeDir);
-  }
-  
-  @Override
-  public String getSolrHome() { 
-    return homeDir.getAbsolutePath(); 
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig.xml", "schema.xml");
   }
 
-  @Override
-  public void setUp() throws Exception {
-    homeDir = new File(TEMP_DIR + File.separator + "solr-test-home-" + System.nanoTime());
-    homeDir.mkdirs();
-    FileUtils.copyDirectory(new File(getFile("solr/" + collection).getParent()), homeDir, false);
-
-    super.setUp();
-    
-    jetty = new JettySolrRunner(getSolrHome(), context, 0 );
-    jetty.start(false);
-    port = jetty.getLocalPort();
-  }
-  
   public void testStopwordsAfterCoreReload() throws Exception {
     SolrInputDocument doc = new SolrInputDocument();
     doc.setField( "id", "42" );
@@ -100,8 +77,7 @@ public class AnalysisAfterCoreReloadTest
 
     // overwrite stopwords file with stopword list ["stopwordc"] and reload the core
     overwriteStopwords("stopwordc\n");
-    SolrServer coreadmin = getSolrAdmin();
-    CoreAdminRequest.reloadCore(collection, coreadmin);
+    h.getCoreContainer().reload(collection);
 
     up.process( getSolrCore() );
 
@@ -133,42 +109,33 @@ public class AnalysisAfterCoreReloadTest
     SolrCore core = h.getCoreContainer().getCore(collection);
     try {
       String configDir = core.getResourceLoader().getConfigDir();
+      FileUtils.moveFile(new File(configDir, "stopwords.txt"), new File(configDir, "stopwords.txt.bak"));
       File file = new File(configDir, "stopwords.txt");
       FileUtils.writeStringToFile(file, stopwords);
+     
     } finally {
       core.close();
     }
   }
   
-  protected SolrServer getSolrAdmin() {
-    return createServer("");
-  }
-  protected SolrServer getSolrCore() {
-    return createServer(collection);
-  }
-  private SolrServer createServer( String name ) {
+  @Override
+  public void tearDown() throws Exception {
+    SolrCore core = h.getCoreContainer().getCore(collection);
+    String configDir;
     try {
-      // setup the server...
-      String url = "http://127.0.0.1:"+port+context+"/"+name;
-      HttpSolrServer s = new HttpSolrServer( url );
-      s.setConnectionTimeout(SolrTestCaseJ4.DEFAULT_CONNECTION_TIMEOUT);
-      s.setDefaultMaxConnectionsPerHost(100);
-      s.setMaxTotalConnections(100);
-      return s;
+      configDir = core.getResourceLoader().getConfigDir();
+    } finally {
+      core.close();
     }
-    catch( Exception ex ) {
-      throw new RuntimeException( ex );
+    super.tearDown();
+    if (new File(configDir, "stopwords.txt.bak").exists()) {
+      FileUtils.deleteQuietly(new File(configDir, "stopwords.txt"));
+      FileUtils.moveFile(new File(configDir, "stopwords.txt.bak"), new File(configDir, "stopwords.txt"));
     }
   }
 
-  @Override
-  public String getSchemaFile() {
-    return "schema.xml";
-  }
-
-  @Override
-  public String getSolrConfigFile() {
-    return "solrconfig.xml";
+  protected SolrServer getSolrCore() {
+    return new EmbeddedSolrServer(h.getCore());
   }
 
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/analysis/TestLuceneMatchVersion.java Wed Dec 12 21:41:06 2012
@@ -52,10 +52,10 @@ public class TestLuceneMatchVersion exte
     assertEquals(DEFAULT_VERSION, (ana.getTokenizerFactory()).getLuceneMatchVersion());
     assertEquals(DEFAULT_VERSION, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion());
 
-    type = schema.getFieldType("text30");
+    type = schema.getFieldType("text40");
     ana = (TokenizerChain) type.getAnalyzer();
-    assertEquals(Version.LUCENE_30, (ana.getTokenizerFactory()).getLuceneMatchVersion());
-    assertEquals(Version.LUCENE_31, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion());
+    assertEquals(Version.LUCENE_40, (ana.getTokenizerFactory()).getLuceneMatchVersion());
+    assertEquals(Version.LUCENE_40, (ana.getTokenFilterFactories()[2]).getLuceneMatchVersion());
 
     // this is a hack to get the private matchVersion field in StandardAnalyzer's superclass, may break in later lucene versions - we have no getter :(
     final Field matchVersionField = StandardAnalyzer.class.getSuperclass().getDeclaredField("matchVersion");
@@ -66,9 +66,9 @@ public class TestLuceneMatchVersion exte
     assertTrue(ana1 instanceof StandardAnalyzer);
     assertEquals(DEFAULT_VERSION, matchVersionField.get(ana1));
 
-    type = schema.getFieldType("textStandardAnalyzer30");
+    type = schema.getFieldType("textStandardAnalyzer40");
     ana1 = type.getAnalyzer();
     assertTrue(ana1 instanceof StandardAnalyzer);
-    assertEquals(Version.LUCENE_30, matchVersionField.get(ana1));
+    assertEquals(Version.LUCENE_40, matchVersionField.get(ana1));
   }
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Wed Dec 12 21:41:06 2012
@@ -33,13 +33,22 @@ import org.apache.solr.common.cloud.ZkSt
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.servlet.SolrDispatchFilter;
+import org.junit.BeforeClass;
 
 /**
  * This test simply does a bunch of basic things in solrcloud mode and asserts things
  * work as expected.
  */
 public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
-
+  @BeforeClass
+  public static void beforeThisClass2() throws Exception {
+    // TODO: we use an fs based dir because something
+    // like a ram dir will not recover correctly right now
+    // because tran log will still exist on restart and ram
+    // dir will not persist - perhaps translog can empty on
+    // start if using an EphemeralDirectoryFactory 
+    useFactory(null);
+  }
   /*
    * (non-Javadoc)
    * 
@@ -100,7 +109,8 @@ public class BasicDistributedZk2Test ext
       
       // TODO: bring this to it's own method?
       // try indexing to a leader that has no replicas up
-      ZkNodeProps leaderProps = zkStateReader.getLeaderProps(
+      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+      ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(
           DEFAULT_COLLECTION, SHARD2);
       
       String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP);
@@ -175,9 +185,13 @@ public class BasicDistributedZk2Test ext
 
     query("q", "*:*", "sort", "n_tl1 desc");
     
+    int oldLiveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
+    
+    assertEquals(5, oldLiveNodes);
+    
     // kill a shard
     CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD2, 0);
-    cloudClient.connect();
+
 
     // we are careful to make sure the downed node is no longer in the state,
     // because on some systems (especially freebsd w/ blackhole enabled), trying
@@ -186,10 +200,23 @@ public class BasicDistributedZk2Test ext
     jetties.addAll(shardToJetty.get(SHARD2));
     jetties.remove(deadShard);
     
+    // wait till live nodes drops by 1
+    int liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
+    int tries = 50;
+    while(oldLiveNodes == liveNodes) {
+      Thread.sleep(100);
+      if (tries-- == 0) {
+        fail("We expected a node to drop...");
+      }
+      liveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
+    }
+    assertEquals(4, liveNodes);
+
+    int cnt = 0;
     for (CloudJettyRunner cjetty : jetties) {
       waitToSeeNotLive(((SolrDispatchFilter) cjetty.jetty.getDispatchFilter()
           .getFilter()).getCores().getZkController().getZkStateReader(),
-          deadShard);
+          deadShard, cnt++);
     }
     waitToSeeNotLive(cloudClient.getZkStateReader(), deadShard);
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Wed Dec 12 21:41:06 2012
@@ -66,6 +66,7 @@ import org.apache.solr.common.SolrExcept
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -82,6 +83,7 @@ import org.apache.solr.update.DirectUpda
 import org.apache.solr.update.SolrCmdDistributor.Request;
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.junit.Before;
+import org.junit.BeforeClass;
 
 /**
  * This test simply does a bunch of basic things in solrcloud mode and asserts things
@@ -121,6 +123,12 @@ public class BasicDistributedZkTest exte
   CompletionService<Request> completionService;
   Set<Future<Request>> pending;
   
+  @BeforeClass
+  public static void beforeThisClass2() throws Exception {
+    // TODO: we use an fs based dir because something
+    // like a ram dir will not recover correctly right now
+    useFactory(null);
+  }
   
   @Before
   @Override
@@ -162,10 +170,10 @@ public class BasicDistributedZkTest exte
   public void doTest() throws Exception {
     // setLoggingLevel(null);
 
-    
+    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
     // make sure we have leaders for each shard
     for (int j = 1; j < sliceCount; j++) {
-      zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+      zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
     }      // make sure we again have leaders for each shard
     
     waitForRecoveriesToFinish(false);
@@ -438,10 +446,10 @@ public class BasicDistributedZkTest exte
     server.request(createCmd);
     
     ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
-
+    
     zkStateReader.updateClusterState(true);
 
-    int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+    int slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
     assertEquals(1, slices);
     
     client = clients.get(1);
@@ -456,7 +464,7 @@ public class BasicDistributedZkTest exte
     server.request(createCmd);
     
     zkStateReader.updateClusterState(true);
-    slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").size();
+    slices = zkStateReader.getClusterState().getCollectionStates().get("unloadcollection").getSlices().size();
     assertEquals(1, slices);
     
     waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
@@ -530,7 +538,7 @@ public class BasicDistributedZkTest exte
     }
     
     // ensure there is a leader
-    zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
     
     addClient = new HttpSolrServer(url2 + "/unloadcollection2");
     // add a few docs while the leader is down
@@ -572,7 +580,7 @@ public class BasicDistributedZkTest exte
       }
     }
     
-    zkStateReader.getLeaderProps("unloadcollection", "shard1", 15000);
+    zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000);
     
     
     // set this back
@@ -677,7 +685,7 @@ public class BasicDistributedZkTest exte
     ChaosMonkey.start(cloudJettys.get(0).jetty);
     cloudClient.getZkStateReader().updateClusterState(true);
     try {
-      cloudClient.getZkStateReader().getLeaderProps("multiunload2", "shard1", 30000);
+      cloudClient.getZkStateReader().getLeaderRetry("multiunload2", "shard1", 30000);
     } catch (SolrException e) {
       printLayout();
       throw e;
@@ -715,6 +723,7 @@ public class BasicDistributedZkTest exte
     }
   }
 
+
   private String getBaseUrl(SolrServer client) {
     String url2 = ((HttpSolrServer) client).getBaseURL()
         .substring(
@@ -756,7 +765,7 @@ public class BasicDistributedZkTest exte
       // poll for a second - it can take a moment before we are ready to serve
       waitForNon403or404or503(collectionClient);
     }
-
+    ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
     for (int j = 0; j < cnt; j++) {
       waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
     }
@@ -942,10 +951,10 @@ public class BasicDistributedZkTest exte
 
   private void collectStartTimes(String collectionName,
       Map<String,Long> urlToTime) throws SolrServerException, IOException {
-    Map<String,Map<String,Slice>> collections = getCommonCloudSolrServer().getZkStateReader()
+    Map<String,DocCollection> collections = getCommonCloudSolrServer().getZkStateReader()
         .getClusterState().getCollectionStates();
     if (collections.containsKey(collectionName)) {
-      Map<String,Slice> slices = collections.get(collectionName);
+      Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
 
       Iterator<Entry<String,Slice>> it = slices.entrySet().iterator();
       while (it.hasNext()) {
@@ -971,7 +980,7 @@ public class BasicDistributedZkTest exte
 
   private String getUrlFromZk(String collection) {
     ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-    Map<String,Slice> slices = clusterState.getCollectionStates().get(collection);
+    Map<String,Slice> slices = clusterState.getCollectionStates().get(collection).getSlicesMap();
     
     if (slices == null) {
       throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
@@ -1035,10 +1044,10 @@ public class BasicDistributedZkTest exte
     int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
     int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
     
-      Map<String,Map<String,Slice>> collections = clusterState
+      Map<String,DocCollection> collections = clusterState
           .getCollectionStates();
       if (collections.containsKey(collectionName)) {
-        Map<String,Slice> slices = collections.get(collectionName);
+        Map<String,Slice> slices = collections.get(collectionName).getSlicesMap();
         // did we find expectedSlices slices/shards?
       if (slices.size() != expectedSlices) {
         return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
@@ -1091,7 +1100,7 @@ public class BasicDistributedZkTest exte
     while (System.currentTimeMillis() < timeoutAt) {
       getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
       ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
-      Map<String,Map<String,Slice>> collections = clusterState
+      Map<String,DocCollection> collections = clusterState
           .getCollectionStates();
       if (!collections.containsKey(collectionName)) {
         found = false;
@@ -1154,6 +1163,7 @@ public class BasicDistributedZkTest exte
 
   // cloud level test mainly needed just to make sure that versions and errors are propagated correctly
   private void doOptimisticLockingAndUpdating() throws Exception {
+    log.info("### STARTING doOptimisticLockingAndUpdating");
     printLayout();
     
     SolrInputDocument sd =  sdoc("id", 1000, "_version_", -1);
@@ -1194,6 +1204,7 @@ public class BasicDistributedZkTest exte
 
   private void testNumberOfCommitsWithCommitAfterAdd()
       throws SolrServerException, IOException {
+    log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd");
     long startCommits = getNumCommits((HttpSolrServer) clients.get(0));
     
     ContentStreamUpdateRequest up = new ContentStreamUpdateRequest("/update");
@@ -1225,6 +1236,7 @@ public class BasicDistributedZkTest exte
   }
 
   private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception {
+    log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement");
     System.clearProperty("numShards");
     List<SolrServer> collectionClients = new ArrayList<SolrServer>();
     SolrServer client = clients.get(0);
@@ -1290,7 +1302,7 @@ public class BasicDistributedZkTest exte
     // we added a role of none on these creates - check for it
     ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
     zkStateReader.updateClusterState(true);
-    Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(oneInstanceCollection2);
+    Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(oneInstanceCollection2);
     assertNotNull(slices);
     String roles = slices.get("slice1").getReplicasMap().values().iterator().next().getStr(ZkStateReader.ROLES_PROP);
     assertEquals("none", roles);
@@ -1318,6 +1330,7 @@ public class BasicDistributedZkTest exte
   }
 
   private void testSearchByCollectionName() throws SolrServerException {
+    log.info("### STARTING testSearchByCollectionName");
     SolrServer client = clients.get(0);
     final String baseUrl = ((HttpSolrServer) client).getBaseURL().substring(
         0,
@@ -1333,6 +1346,7 @@ public class BasicDistributedZkTest exte
   }
 
   private void testANewCollectionInOneInstance() throws Exception {
+    log.info("### STARTING testANewCollectionInOneInstance");
     List<SolrServer> collectionClients = new ArrayList<SolrServer>();
     SolrServer client = clients.get(0);
     otherCollectionClients.put(oneInstanceCollection , collectionClients);
@@ -1428,6 +1442,7 @@ public class BasicDistributedZkTest exte
   }
 
   private void testMultipleCollections() throws Exception {
+    log.info("### STARTING testMultipleCollections");
     // create another 2 collections and search across them
     createNewCollection("collection2");
     createNewCollection("collection3");
@@ -1441,30 +1456,30 @@ public class BasicDistributedZkTest exte
     
     indexDoc("collection2", getDoc(id, "10000000")); 
     indexDoc("collection2", getDoc(id, "10000001")); 
-    indexDoc("collection2", getDoc(id, "10000003")); 
-    
+    indexDoc("collection2", getDoc(id, "10000003"));
     getCommonCloudSolrServer().setDefaultCollection("collection2");
     getCommonCloudSolrServer().add(getDoc(id, "10000004"));
     getCommonCloudSolrServer().setDefaultCollection(null);
     
     indexDoc("collection3", getDoc(id, "20000000"));
     indexDoc("collection3", getDoc(id, "20000001")); 
-
     getCommonCloudSolrServer().setDefaultCollection("collection3");
     getCommonCloudSolrServer().add(getDoc(id, "10000005"));
     getCommonCloudSolrServer().setDefaultCollection(null);
     
     otherCollectionClients.get("collection2").get(0).commit();
     otherCollectionClients.get("collection3").get(0).commit();
-
+    
     getCommonCloudSolrServer().setDefaultCollection("collection1");
     long collection1Docs = getCommonCloudSolrServer().query(new SolrQuery("*:*")).getResults()
-
         .getNumFound();
+
     long collection2Docs = otherCollectionClients.get("collection2").get(0)
         .query(new SolrQuery("*:*")).getResults().getNumFound();
+    System.out.println("found2: "+ collection2Docs);
     long collection3Docs = otherCollectionClients.get("collection3").get(0)
         .query(new SolrQuery("*:*")).getResults().getNumFound();
+    System.out.println("found3: "+ collection3Docs);
     
     SolrQuery query = new SolrQuery("*:*");
     query.set("collection", "collection2,collection3");
@@ -1491,6 +1506,8 @@ public class BasicDistributedZkTest exte
     query.remove("collection");
     found = getCommonCloudSolrServer().query(query).getResults().getNumFound();
     assertEquals(collection1Docs, found);
+    
+    assertEquals(collection3Docs, collection2Docs - 1);
   }
   
   private void checkNoTwoShardsUseTheSameIndexDir() throws Exception {

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java Wed Dec 12 21:41:06 2012
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.LogMergePolicy;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -32,7 +33,6 @@ import org.apache.solr.util.RefCounted;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.xml.sax.SAXParseException;
 
 /**
  * This test is not fully functional - the port registered is illegal - 
@@ -148,19 +148,19 @@ public class BasicZkTest extends Abstrac
     assertU(delQ("id:[100 TO 110]"));
     assertU(commit());
     assertQ(request("id:[100 TO 110]"), "//*[@numFound='0']");
-    
-   
-    
+
+
+
     // SOLR-2651: test that reload still gets config files from zookeeper 
     zkController.getZkClient().setData("/configs/conf1/solrconfig.xml", new byte[0], true);
  
     // we set the solrconfig to nothing, so this reload should fail
     try {
-      SolrTestCaseJ4.ignoreException("SAXParseException");
+      SolrTestCaseJ4.ignoreException("SolrException");
       h.getCoreContainer().reload(h.getCore().getName());
       SolrTestCaseJ4.resetExceptionIgnores();
       fail("The reloaded SolrCore did not pick up configs from zookeeper");
-    } catch(SAXParseException e) {
+    } catch(SolrException e) {
       
     }
     

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java Wed Dec 12 21:41:06 2012
@@ -29,6 +29,7 @@ import org.apache.solr.client.solrj.impl
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -82,10 +83,10 @@ public class ChaosMonkeyNothingIsSafeTes
       handle.clear();
       handle.put("QTime", SKIPVAL);
       handle.put("timestamp", SKIPVAL);
-      
+      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
       // make sure we have leaders for each shard
       for (int j = 1; j < sliceCount; j++) {
-        zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+        zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
       }      // make sure we again have leaders for each shard
       
       waitForRecoveriesToFinish(false);
@@ -155,7 +156,7 @@ public class ChaosMonkeyNothingIsSafeTes
       
       // make sure we again have leaders for each shard
       for (int j = 1; j < sliceCount; j++) {
-        zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard" + j, 10000);
+        zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
       }
       
       commit();
@@ -189,19 +190,6 @@ public class ChaosMonkeyNothingIsSafeTes
       }
     }
   }
-  
-  // skip the randoms - they can deadlock...
-  protected void indexr(Object... fields) throws Exception {
-    SolrInputDocument doc = getDoc(fields);
-    indexDoc(doc);
-  }
-
-  private SolrInputDocument getDoc(Object... fields) {
-    SolrInputDocument doc = new SolrInputDocument();
-    addFields(doc, fields);
-    addFields(doc, "rnd_b", true);
-    return doc;
-  }
 
   class FullThrottleStopableIndexingThread extends StopableIndexingThread {
     private HttpClient httpClient = HttpClientUtil.createClient(null);
@@ -219,6 +207,7 @@ public class ChaosMonkeyNothingIsSafeTes
       suss = new ConcurrentUpdateSolrServer(
           ((HttpSolrServer) clients.get(0)).getBaseURL(), httpClient, 8,
           2) {
+        @Override
         public void handleError(Throwable ex) {
           log.warn("suss error", ex);
         }
@@ -287,6 +276,7 @@ public class ChaosMonkeyNothingIsSafeTes
         suss = new ConcurrentUpdateSolrServer(
             ((HttpSolrServer) clients.get(clientIndex)).getBaseURL(),
             httpClient, 30, 3) {
+          @Override
           public void handleError(Throwable ex) {
             log.warn("suss error", ex);
           }
@@ -294,16 +284,33 @@ public class ChaosMonkeyNothingIsSafeTes
       }
     }
     
+    @Override
     public void safeStop() {
       stop = true;
       suss.shutdownNow();
       httpClient.getConnectionManager().shutdown();
     }
 
+    @Override
     public int getFails() {
       return fails.get();
     }
     
   };
   
+  
+  // skip the randoms - they can deadlock...
+  @Override
+  protected void indexr(Object... fields) throws Exception {
+    SolrInputDocument doc = getDoc(fields);
+    indexDoc(doc);
+  }
+
+  SolrInputDocument getDoc(Object... fields) throws Exception {
+    SolrInputDocument doc = new SolrInputDocument();
+    addFields(doc, fields);
+    addFields(doc, "rnd_b", true);
+    return doc;
+  }
+  
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateTest.java Wed Dec 12 21:41:06 2012
@@ -24,6 +24,9 @@ import java.util.Set;
 
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.ZkStateReader;
@@ -32,7 +35,7 @@ import org.junit.Test;
 public class ClusterStateTest extends SolrTestCaseJ4 {
   @Test
   public void testStoreAndRead() throws Exception {
-    Map<String,Map<String,Slice>> collectionStates = new HashMap<String,Map<String,Slice>>();
+    Map<String,DocCollection> collectionStates = new HashMap<String,DocCollection>();
     Set<String> liveNodes = new HashSet<String>();
     liveNodes.add("node1");
     liveNodes.add("node2");
@@ -49,12 +52,12 @@ public class ClusterStateTest extends So
     slices.put("shard1", slice);
     Slice slice2 = new Slice("shard2", sliceToProps, null);
     slices.put("shard2", slice2);
-    collectionStates.put("collection1", slices);
-    collectionStates.put("collection2", slices);
+    collectionStates.put("collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
+    collectionStates.put("collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
     
     ClusterState clusterState = new ClusterState(liveNodes, collectionStates);
     byte[] bytes = ZkStateReader.toJSON(clusterState);
-    
+    // System.out.println("#################### " + new String(bytes));
     ClusterState loadedClusterState = ClusterState.load(null, bytes, liveNodes);
     
     assertEquals("Provided liveNodes not used properly", 2, loadedClusterState

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java Wed Dec 12 21:41:06 2012
@@ -171,7 +171,7 @@ public class ClusterStateUpdateTest exte
     Map<String,Slice> slices = null;
     for (int i = 75; i > 0; i--) {
       clusterState2 = zkController2.getClusterState();
-      slices = clusterState2.getSlices("testcore");
+      slices = clusterState2.getSlicesMap("testcore");
       
       if (slices != null && slices.containsKey("shard1")
           && slices.get("shard1").getReplicasMap().size() > 0) {

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Wed Dec 12 21:41:06 2012
@@ -131,7 +131,8 @@ public class FullSolrCloudDistribCmdsTes
   }
 
   private void testThatCantForwardToLeaderFails() throws Exception {
-    ZkNodeProps props = zkStateReader.getLeaderProps(DEFAULT_COLLECTION, "shard1");
+    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+    ZkNodeProps props = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard1");
     
     chaosMonkey.stopShard("shard1");
 
@@ -250,7 +251,6 @@ public class FullSolrCloudDistribCmdsTes
   
   private void testOptimisticUpdate(QueryResponse results) throws Exception {
     SolrDocument doc = results.getResults().get(0);
-    System.out.println("version:" + doc.getFieldValue(VersionInfo.VERSION_FIELD));
     Long version = (Long) doc.getFieldValue(VersionInfo.VERSION_FIELD);
     Integer theDoc = (Integer) doc.getFieldValue("id");
     UpdateRequest uReq = new UpdateRequest();

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java Wed Dec 12 21:41:06 2012
@@ -256,7 +256,7 @@ public class LeaderElectionIntegrationTe
   
   private String getLeader() throws InterruptedException {
     
-    ZkNodeProps props = reader.getLeaderProps("collection1", "shard1", 30000);
+    ZkNodeProps props = reader.getLeaderRetry("collection1", "shard1", 30000);
     String leader = props.getStr(ZkStateReader.NODE_NAME_PROP);
     
     return leader;

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Wed Dec 12 21:41:06 2012
@@ -35,6 +35,7 @@ import java.util.Set;
 
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CoreAdminParams;
@@ -61,6 +62,7 @@ public class OverseerCollectionProcessor
   private ShardHandler shardHandlerMock;
   private ZkStateReader zkStateReaderMock;
   private ClusterState clusterStateMock;
+  private SolrZkClient solrZkClientMock;
   
   private Thread thread;
   private Queue<byte[]> queue = new BlockingArrayQueue<byte[]>();
@@ -88,6 +90,7 @@ public class OverseerCollectionProcessor
     shardHandlerMock = createMock(ShardHandler.class);
     zkStateReaderMock = createMock(ZkStateReader.class);
     clusterStateMock = createMock(ClusterState.class);
+    solrZkClientMock = createMock(SolrZkClient.class);
     underTest = new OverseerCollectionProcessorToBeTested(zkStateReaderMock,
         "1234", shardHandlerMock, ADMIN_PATH, workQueueMock);
   }
@@ -129,6 +132,15 @@ public class OverseerCollectionProcessor
       }
     }).anyTimes();
     
+    zkStateReaderMock.getZkClient();
+    expectLastCall().andAnswer(new IAnswer<Object>() {
+      @Override
+      public Object answer() throws Throwable {
+        return solrZkClientMock;
+      }
+    }).anyTimes();
+    
+    
     clusterStateMock.getCollections();
     expectLastCall().andAnswer(new IAnswer<Object>() {
       @Override
@@ -138,7 +150,19 @@ public class OverseerCollectionProcessor
     }).anyTimes();
     final Set<String> liveNodes = new HashSet<String>();
     for (int i = 0; i < liveNodesCount; i++) {
-      liveNodes.add("localhost:" + (8963 + i) + "_solr");
+      final String address = "localhost:" + (8963 + i) + "_solr";
+      liveNodes.add(address);
+      
+      solrZkClientMock.getBaseUrlForNodeName(address);
+      expectLastCall().andAnswer(new IAnswer<Object>() {
+        @Override
+        public Object answer() throws Throwable {
+          // This works as long as this test does not use a 
+          // webapp context with an underscore in it
+          return address.replaceAll("_", "/");
+        }
+      }).anyTimes();
+      
     }
     clusterStateMock.getLiveNodes();
     expectLastCall().andAnswer(new IAnswer<Object>() {
@@ -336,6 +360,7 @@ public class OverseerCollectionProcessor
     }
     
     replay(workQueueMock);
+    replay(solrZkClientMock);
     replay(zkStateReaderMock);
     replay(clusterStateMock);
     replay(shardHandlerMock);

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Wed Dec 12 21:41:06 2012
@@ -35,6 +35,7 @@ import javax.xml.parsers.ParserConfigura
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.SolrZkClient;
 import org.apache.solr.common.cloud.ZkNodeProps;
@@ -148,7 +149,7 @@ public class OverseerTest extends SolrTe
     }
     
     private String getShardId(final String coreName) {
-      Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(
+      Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(
           collection);
       if (slices != null) {
         for (Slice slice : slices.values()) {
@@ -301,7 +302,7 @@ public class OverseerTest extends SolrTe
         cloudStateSliceCount = 0;
         reader.updateClusterState(true);
         ClusterState state = reader.getClusterState();
-        Map<String,Slice> slices = state.getSlices("collection1");
+        Map<String,Slice> slices = state.getSlicesMap("collection1");
         for (String name : slices.keySet()) {
           cloudStateSliceCount += slices.get(name).getReplicasMap().size();
         }
@@ -478,7 +479,7 @@ public class OverseerTest extends SolrTe
   }
   
   private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException {
-    int maxIterations = 100;
+    int maxIterations = 200;
     while(maxIterations-->0) {
       reader.updateClusterState(true); // poll state
       ZkNodeProps props =  reader.getClusterState().getLeader(collection, shard);
@@ -712,8 +713,8 @@ public class OverseerTest extends SolrTe
       ClusterState state = reader.getClusterState();
       
       int numFound = 0;
-      for (Map<String,Slice> collection : state.getCollectionStates().values()) {
-        for (Slice slice : collection.values()) {
+      for (DocCollection collection : state.getCollectionStates().values()) {
+        for (Slice slice : collection.getSlices()) {
           if (slice.getReplicasMap().get("node1_core1") != null) {
             numFound++;
           }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Wed Dec 12 21:41:06 2012
@@ -24,6 +24,7 @@ import org.apache.solr.client.solrj.Solr
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -71,7 +72,7 @@ public class RecoveryZkTest extends Abst
     
     // make sure replication can start
     Thread.sleep(1500);
-    
+    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
     waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
     
     // stop indexing threads

Added: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java?rev=1420992&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java (added)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingTest.java Wed Dec 12 21:41:06 2012
@@ -0,0 +1,248 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.CompositeIdRouter;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.update.DirectUpdateHandler2;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+
+public class ShardRoutingTest extends AbstractFullDistribZkTestBase {
+  @BeforeClass
+  public static void beforeShardHashingTest() throws Exception {
+    // TODO: we use an fs based dir because something
+    // like a ram dir will not recover correctly right now
+    // because tran log will still exist on restart and ram
+    // dir will not persist - perhaps translog can empty on
+    // start if using an EphemeralDirectoryFactory 
+    useFactory(null);
+  }
+
+  public ShardRoutingTest() {
+    schemaString = "schema15.xml";      // we need a string id
+    super.sliceCount = 4;
+    super.shardCount = 8;
+    super.fixShardCount = true;  // we only want to test with exactly 4 slices.
+
+    // from negative to positive, the upper bits of the hash ranges should be
+    // shard1: top bits:10  80000000:bfffffff
+    // shard2: top bits:11  c0000000:ffffffff
+    // shard3: top bits:00  00000000:3fffffff
+    // shard4: top bits:01  40000000:7fffffff
+
+    /***
+     hash of a is 3c2569b2 high bits=0 shard=shard3
+     hash of b is 95de7e03 high bits=2 shard=shard1
+     hash of c is e132d65f high bits=3 shard=shard2
+     hash of d is 27191473 high bits=0 shard=shard3
+     hash of e is 656c4367 high bits=1 shard=shard4
+     hash of f is 2b64883b high bits=0 shard=shard3
+     hash of g is f18ae416 high bits=3 shard=shard2
+     hash of h is d482b2d3 high bits=3 shard=shard2
+     hash of i is 811a702b high bits=2 shard=shard1
+     hash of j is ca745a39 high bits=3 shard=shard2
+     hash of k is cfbda5d1 high bits=3 shard=shard2
+     hash of l is 1d5d6a2c high bits=0 shard=shard3
+     hash of m is 5ae4385c high bits=1 shard=shard4
+     hash of n is c651d8ac high bits=3 shard=shard2
+     hash of o is 68348473 high bits=1 shard=shard4
+     hash of p is 986fdf9a high bits=2 shard=shard1
+     hash of q is ff8209e8 high bits=3 shard=shard2
+     hash of r is 5c9373f1 high bits=1 shard=shard4
+     hash of s is ff4acaf1 high bits=3 shard=shard2
+     hash of t is ca87df4d high bits=3 shard=shard2
+     hash of u is 62203ae0 high bits=1 shard=shard4
+     hash of v is bdafcc55 high bits=2 shard=shard1
+     hash of w is ff439d1f high bits=3 shard=shard2
+     hash of x is 3e9a9b1b high bits=0 shard=shard3
+     hash of y is 477d9216 high bits=1 shard=shard4
+     hash of z is c1f69a17 high bits=3 shard=shard2
+     ***/
+  }
+
+  @Override
+  public void doTest() throws Exception {
+    boolean testFinished = false;
+    try {
+      handle.clear();
+      handle.put("QTime", SKIPVAL);
+      handle.put("timestamp", SKIPVAL);
+
+      // todo: do I have to do this here?
+      waitForRecoveriesToFinish(false);
+
+      doHashingTest();
+
+      testFinished = true;
+    } finally {
+      if (!testFinished) {
+        printLayoutOnTearDown = true;
+      }
+    }
+  }
+
+
+
+
+  private void doHashingTest() throws Exception {
+    assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
+    String shardKeys = ShardParams.SHARD_KEYS;
+    // for now,  we know how ranges will be distributed to shards.
+    // may have to look it up in clusterstate if that assumption changes.
+
+    String bucket1 = "shard1";      // shard1: top bits:10  80000000:bfffffff
+    String bucket2 = "shard2";      // shard2: top bits:11  c0000000:ffffffff
+    String bucket3 = "shard3";      // shard3: top bits:00  00000000:3fffffff
+    String bucket4 = "shard4";      // shard4: top bits:01  40000000:7fffffff
+
+    doAddDoc("b!doc1");
+    doAddDoc("c!doc2");
+    doAddDoc("d!doc3");
+    doAddDoc("e!doc4");
+
+    doRTG("b!doc1");
+    doRTG("c!doc2");
+    doRTG("d!doc3");
+    doRTG("e!doc4");
+    doRTG("b!doc1,c!doc2");
+    doRTG("d!doc3,e!doc4");
+
+    commit();
+
+    doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*");
+    doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", "shards","shard1,shard2,shard3,shard4");
+    doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", shardKeys,"b!,c!,d!,e!");
+    doQuery("b!doc1", "q","*:*", shardKeys,"b!");
+    doQuery("c!doc2", "q","*:*", shardKeys,"c!");
+    doQuery("d!doc3", "q","*:*", shardKeys,"d!");
+    doQuery("e!doc4", "q","*:*", shardKeys,"e!");
+
+    // try using shards parameter
+    doQuery("b!doc1", "q","*:*", "shards",bucket1);
+    doQuery("c!doc2", "q","*:*", "shards",bucket2);
+    doQuery("d!doc3", "q","*:*", "shards",bucket3);
+    doQuery("e!doc4", "q","*:*", "shards",bucket4);
+
+
+    doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b!,c!");
+    doQuery("b!doc1,e!doc4", "q","*:*", shardKeys,"b!,e!");
+
+    doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b,c");     // query shards that would contain *documents* "b" and "c" (i.e. not prefixes).  The upper bits are the same, so the shards should be the same.
+
+    doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b/1!");   // top bit of hash(b)==1, so shard1 and shard2
+    doQuery("d!doc3,e!doc4", "q","*:*", shardKeys,"d/1!");   // top bit of hash(b)==0, so shard3 and shard4
+
+    doQuery("b!doc1,c!doc2", "q","*:*", shardKeys,"b!,c!");
+
+    doQuery("b!doc1,c!doc2,d!doc3,e!doc4", "q","*:*", shardKeys,"foo/0!");
+
+    // test targeting deleteByQuery at only certain shards
+    doDBQ("*:*", shardKeys,"b!");
+    commit();
+    doQuery("c!doc2,d!doc3,e!doc4", "q","*:*");
+    doAddDoc("b!doc1");
+
+    doDBQ("*:*", shardKeys,"c!");
+    commit();
+    doQuery("b!doc1,d!doc3,e!doc4", "q","*:*");
+    doAddDoc("c!doc2");
+
+    doDBQ("*:*", shardKeys,"c!");
+    commit();
+    doQuery("b!doc1,d!doc3,e!doc4", "q","*:*");
+    doAddDoc("c!doc2");
+
+    doDBQ("*:*", shardKeys,"d!,e!");
+    commit();
+    doQuery("b!doc1,c!doc2", "q","*:*");
+    doAddDoc("d!doc3");
+    doAddDoc("e!doc4");
+
+    commit();
+
+
+  }
+
+  void doAddDoc(String id) throws Exception {
+    index("id",id);
+    // todo - target diff servers and use cloud clients as well as non-cloud clients
+  }
+
+  // TODO: refactor some of this stuff up into a base class for use by other tests
+  void doQuery(String expectedDocs, String... queryParams) throws Exception {
+    Set<String> expectedIds = new HashSet<String>( StrUtils.splitSmart(expectedDocs, ",", true) );
+
+    QueryResponse rsp = cloudClient.query(params(queryParams));
+    Set<String> obtainedIds = new HashSet<String>();
+    for (SolrDocument doc : rsp.getResults()) {
+      obtainedIds.add((String) doc.get("id"));
+    }
+
+    assertEquals(expectedIds, obtainedIds);
+  }
+
+  void doRTG(String ids) throws Exception {
+    cloudClient.query(params("qt","/get", "ids",ids));
+
+    Set<String> expectedIds = new HashSet<String>( StrUtils.splitSmart(ids, ",", true) );
+
+    QueryResponse rsp = cloudClient.query(params("qt","/get", "ids",ids));
+    Set<String> obtainedIds = new HashSet<String>();
+    for (SolrDocument doc : rsp.getResults()) {
+      obtainedIds.add((String) doc.get("id"));
+    }
+
+    assertEquals(expectedIds, obtainedIds);
+  }
+
+  // TODO: refactor some of this stuff into the SolrJ client... it should be easier to use
+  void doDBQ(String q, String... reqParams) throws Exception {
+    UpdateRequest req = new UpdateRequest();
+    req.deleteByQuery(q);
+    req.setParams(params(reqParams));
+    req.process(cloudClient);
+  }
+
+  @Override
+  public void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+}

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Wed Dec 12 21:41:06 2012
@@ -47,8 +47,11 @@ import org.junit.BeforeClass;
 public class SyncSliceTest extends AbstractFullDistribZkTestBase {
   
   @BeforeClass
-  public static void beforeSuperClass() {
-    
+  public static void beforeSuperClass() throws Exception {
+    // TODO: we use an fs based dir because something
+    // like a ram dir will not recovery correctly right now
+    // due to tran log persisting across restarts
+    useFactory(null);
   }
   
   @AfterClass

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestHashPartitioner.java Wed Dec 12 21:41:06 2012
@@ -17,16 +17,26 @@ package org.apache.solr.cloud;
  * the License.
  */
 
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.cloud.HashPartitioner;
-import org.apache.solr.common.cloud.HashPartitioner.Range;
+import org.apache.solr.common.cloud.CompositeIdRouter;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.DocRouter.Range;
+import org.apache.solr.common.cloud.PlainIdRouter;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.util.Hash;
+import org.apache.solr.common.util.StrUtils;
 
 public class TestHashPartitioner extends SolrTestCaseJ4 {
   
   public void testMapHashes() throws Exception {
-    HashPartitioner hp = new HashPartitioner();
+    DocRouter hp = DocRouter.DEFAULT;
     List<Range> ranges;
 
     // make sure the partitioner uses the "natural" boundaries and doesn't suffer from an off-by-one
@@ -37,13 +47,22 @@ public class TestHashPartitioner extends
     assertEquals(0x00000000, ranges.get(1).min);
     assertEquals(0x7fffffff, ranges.get(1).max);
 
-    ranges = hp.partitionRange(2, 0, 0x7fffffff);
+    ranges = hp.partitionRange(2, new DocRouter.Range(0, 0x7fffffff));
     assertEquals(0x00000000, ranges.get(0).min);
     assertEquals(0x3fffffff, ranges.get(0).max);
     assertEquals(0x40000000, ranges.get(1).min);
     assertEquals(0x7fffffff, ranges.get(1).max);
 
-    for (int i = 1; i <= 30000; i += 13) {
+    int defaultLowerBits = 0x0000ffff;
+
+    for (int i = 1; i <= 30000; i++) {
+      // start skipping at higher numbers
+      if (i > 100) i+=13;
+      else if (i > 1000) i+=31;
+      else if (i > 5000) i+=101;
+
+      long rangeSize = 0x0000000100000000L / i;
+
       ranges = hp.partitionRange(i, hp.fullRange());
       assertEquals(i, ranges.size());
       assertTrue("First range does not start before " + Integer.MIN_VALUE
@@ -59,8 +78,196 @@ public class TestHashPartitioner extends
         assertEquals(range, newRange);
       }
 
+      // ensure that ranges are contiguous and that size deviations are not too large.
+      int lastEnd = Integer.MIN_VALUE - 1;
+      for (Range range : ranges) {
+        int currStart = range.min;
+        int currEnd = range.max;
+        assertEquals(lastEnd+1, currStart);
+
+        if (ranges.size() < 4000) {
+          // ranges should be rounded to avoid crossing hash domains
+          assertEquals(defaultLowerBits, currEnd & defaultLowerBits);
+
+          // given our rounding condition that domains should be less than 1/16 of the step size,
+          // this means that any sizing deviations should also be less than 1/16th of the idealized range size.
+          // boolean round = rangeStep >= (1<<bits)*16;
+
+          long currRangeSize = (long)currEnd - (long)currStart;
+          long error = Math.abs(rangeSize - currRangeSize);
+          assertTrue( error < rangeSize/16);
+        }
+
+
+        // String s = range.toString();
+        // Range newRange = hp.fromString(s);
+        // assertEquals(range, newRange);
+        lastEnd = currEnd;
+      }
 
     }
   }
-  
+
+  public int hash(String id) {
+    // our hashing is defined to be murmurhash3 on the UTF-8 bytes of the key.
+    return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+  }
+
+  public void testHashCodes() throws Exception {
+    DocRouter router = DocRouter.getDocRouter(PlainIdRouter.NAME);
+    assertTrue(router instanceof PlainIdRouter);
+    DocCollection coll = createCollection(4, router);
+    doNormalIdHashing(coll);
+  }
+
+  public void doNormalIdHashing(DocCollection coll) throws Exception {
+    assertEquals(4, coll.getSlices().size());
+
+    doId(coll, "b", "shard1");
+    doId(coll, "c", "shard2");
+    doId(coll, "d", "shard3");
+    doId(coll, "e", "shard4");
+  }
+
+  public void doId(DocCollection coll, String id, String expectedShard) {
+    doIndex(coll, id, expectedShard);
+    doQuery(coll, id, expectedShard);
+  }
+
+  public void doIndex(DocCollection coll, String id, String expectedShard) {
+    DocRouter router = coll.getRouter();
+    Slice target = router.getTargetSlice(id, null, null, coll);
+    assertEquals(expectedShard, target.getName());
+  }
+
+  public void doQuery(DocCollection coll, String id, String expectedShards) {
+    DocRouter router = coll.getRouter();
+    Collection<Slice> slices = router.getSearchSlices(id, null, coll);
+
+    List<String> expectedShardStr = StrUtils.splitSmart(expectedShards, ",", true);
+
+    HashSet<String> expectedSet = new HashSet<String>(expectedShardStr);
+    HashSet<String> obtainedSet = new HashSet<String>();
+    for (Slice slice : slices) {
+      obtainedSet.add(slice.getName());
+    }
+
+    assertEquals(slices.size(), obtainedSet.size());  // make sure no repeated slices
+    assertEquals(expectedSet, obtainedSet);
+  }
+
+  public void testCompositeHashCodes() throws Exception {
+    DocRouter router = DocRouter.getDocRouter(CompositeIdRouter.NAME);
+    assertTrue(router instanceof CompositeIdRouter);
+    router = DocRouter.DEFAULT;
+    assertTrue(router instanceof CompositeIdRouter);
+
+    DocCollection coll = createCollection(4, router);
+    doNormalIdHashing(coll);
+
+    // ensure that the shard hashed to is only dependent on the first part of the compound key
+    doId(coll, "b!foo", "shard1");
+    doId(coll, "c!bar", "shard2");
+    doId(coll, "d!baz", "shard3");
+    doId(coll, "e!qux", "shard4");
+
+    // syntax to specify bits.
+    // Anything over 2 bits should give the same results as above (since only top 2 bits
+    // affect our 4 slice collection).
+    doId(coll, "b/2!foo", "shard1");
+    doId(coll, "c/2!bar", "shard2");
+    doId(coll, "d/2!baz", "shard3");
+    doId(coll, "e/2!qux", "shard4");
+
+    doId(coll, "b/32!foo", "shard1");
+    doId(coll, "c/32!bar", "shard2");
+    doId(coll, "d/32!baz", "shard3");
+    doId(coll, "e/32!qux", "shard4");
+
+    // no bits allocated to the first part (kind of odd why anyone would do that though)
+    doIndex(coll, "foo/0!b", "shard1");
+    doIndex(coll, "foo/0!c", "shard2");
+    doIndex(coll, "foo/0!d", "shard3");
+    doIndex(coll, "foo/0!e", "shard4");
+
+    // means cover whole range on the query side
+    doQuery(coll, "foo/0!", "shard1,shard2,shard3,shard4");
+
+    doQuery(coll, "b/1!", "shard1,shard2");   // top bit of hash(b)==1, so shard1 and shard2
+    doQuery(coll, "d/1!", "shard3,shard4");   // top bit of hash(b)==0, so shard3 and shard4
+  }
+
+  /***
+  public void testPrintHashCodes() throws Exception {
+   // from negative to positive, the upper bits of the hash ranges should be
+   // shard1: 11
+   // shard2: 10
+   // shard3: 00
+   // shard4: 01
+
+   String[] highBitsToShard = {"shard3","shard4","shard1","shard2"};
+
+
+   for (int i = 0; i<26; i++) {
+      String id  = new String(Character.toChars('a'+i));
+      int hash = hash(id);
+      System.out.println("hash of " + id + " is " + Integer.toHexString(hash) + " high bits=" + (hash>>>30)
+          + " shard="+highBitsToShard[hash>>>30]);
+    }
+  }
+  ***/
+
+
+
+  DocCollection createCollection(int nSlices, DocRouter router) {
+    List<Range> ranges = router.partitionRange(nSlices, router.fullRange());
+
+    Map<String,Slice> slices = new HashMap<String,Slice>();
+    for (int i=0; i<ranges.size(); i++) {
+      Range range = ranges.get(i);
+      Slice slice = new Slice("shard"+(i+1), null, map("range",range));
+      slices.put(slice.getName(), slice);
+    }
+
+    DocCollection coll = new DocCollection("collection1", slices, null, router);
+    return coll;
+  }
+
+
+
+  // from negative to positive, the upper bits of the hash ranges should be
+  // shard1: top bits:10  80000000:bfffffff
+  // shard2: top bits:11  c0000000:ffffffff
+  // shard3: top bits:00  00000000:3fffffff
+  // shard4: top bits:01  40000000:7fffffff
+
+  /***
+   hash of a is 3c2569b2 high bits=0 shard=shard3
+   hash of b is 95de7e03 high bits=2 shard=shard1
+   hash of c is e132d65f high bits=3 shard=shard2
+   hash of d is 27191473 high bits=0 shard=shard3
+   hash of e is 656c4367 high bits=1 shard=shard4
+   hash of f is 2b64883b high bits=0 shard=shard3
+   hash of g is f18ae416 high bits=3 shard=shard2
+   hash of h is d482b2d3 high bits=3 shard=shard2
+   hash of i is 811a702b high bits=2 shard=shard1
+   hash of j is ca745a39 high bits=3 shard=shard2
+   hash of k is cfbda5d1 high bits=3 shard=shard2
+   hash of l is 1d5d6a2c high bits=0 shard=shard3
+   hash of m is 5ae4385c high bits=1 shard=shard4
+   hash of n is c651d8ac high bits=3 shard=shard2
+   hash of o is 68348473 high bits=1 shard=shard4
+   hash of p is 986fdf9a high bits=2 shard=shard1
+   hash of q is ff8209e8 high bits=3 shard=shard2
+   hash of r is 5c9373f1 high bits=1 shard=shard4
+   hash of s is ff4acaf1 high bits=3 shard=shard2
+   hash of t is ca87df4d high bits=3 shard=shard2
+   hash of u is 62203ae0 high bits=1 shard=shard4
+   hash of v is bdafcc55 high bits=2 shard=shard1
+   hash of w is ff439d1f high bits=3 shard=shard2
+   hash of x is 3e9a9b1b high bits=0 shard=shard3
+   hash of y is 477d9216 high bits=1 shard=shard4
+   hash of z is c1f69a17 high bits=3 shard=shard2
+   ***/
+
 }

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Wed Dec 12 21:41:06 2012
@@ -49,6 +49,62 @@ public class ZkControllerTest extends So
     initCore();
   }
 
+  public void testNodeNameUrlConversion() throws Exception {
+
+    // nodeName from parts
+    assertEquals("localhost:8888_solr",
+                 ZkController.generateNodeName("localhost", "8888", "solr"));
+    assertEquals("localhost:8888_", // root context
+                 ZkController.generateNodeName("localhost", "8888", ""));
+    assertEquals("foo-bar:77_solr%2Fsub_dir",
+                 ZkController.generateNodeName("foo-bar", "77", "solr/sub_dir"));
+
+    // setup a SolrZkClient to do some getBaseUrlForNodeName testing
+    String zkDir = dataDir.getAbsolutePath() + File.separator
+        + "zookeeper/server1/data";
+
+    ZkTestServer server = new ZkTestServer(zkDir);
+    try {
+      server.run();
+
+      AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
+      AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
+
+      SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
+      try {
+
+        // getBaseUrlForNodeName
+        assertEquals("http://zzz.xxx:1234/solr",
+                     zkClient.getBaseUrlForNodeName("zzz.xxx:1234_solr"));
+        assertEquals("http://xxx:99/",
+                     zkClient.getBaseUrlForNodeName("xxx:99_"));
+        assertEquals("http://foo-bar.baz.org:9999/some_dir",
+                     zkClient.getBaseUrlForNodeName("foo-bar.baz.org:9999_some_dir"));
+        assertEquals("http://foo-bar.baz.org:9999/solr/sub_dir",
+                     zkClient.getBaseUrlForNodeName("foo-bar.baz.org:9999_solr%2Fsub_dir"));
+        
+        
+        // generateNodeName + getBaseUrlForNodeName
+        assertEquals("http://foo:9876/solr",
+                     zkClient.getBaseUrlForNodeName
+                     (ZkController.generateNodeName("foo","9876","solr")));
+        assertEquals("http://foo.bar.com:9876/solr/sub_dir",
+                     zkClient.getBaseUrlForNodeName
+                     (ZkController.generateNodeName("foo.bar.com","9876","solr/sub_dir")));
+        assertEquals("http://foo-bar:9876/",
+                     zkClient.getBaseUrlForNodeName
+                     (ZkController.generateNodeName("foo-bar","9876","")));
+        assertEquals("http://foo-bar.com:80/some_dir",
+                     zkClient.getBaseUrlForNodeName
+                     (ZkController.generateNodeName("foo-bar.com","80","some_dir")));
+      } finally {
+        zkClient.close();
+      }
+    } finally {
+      server.shutdown();
+    }
+  }
+
   @Test
   public void testReadConfigName() throws Exception {
     String zkDir = dataDir.getAbsolutePath() + File.separator

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java?rev=1420992&r1=1420991&r2=1420992&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java Wed Dec 12 21:41:06 2012
@@ -19,23 +19,26 @@ package org.apache.solr.cloud;
 
 import java.io.File;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import junit.framework.Assert;
-import junit.framework.TestCase;
 
 import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.core.SolrConfig;
 import org.apache.solr.util.AbstractSolrTestCase;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.junit.AfterClass;
+import org.junit.BeforeClass;
 
 public class ZkSolrClientTest extends AbstractSolrTestCase {
   private static final boolean DEBUG = false;
 
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig.xml", "schema.xml");
+  }
+  
   public void testConnect() throws Exception {
     String zkDir = dataDir.getAbsolutePath() + File.separator
         + "zookeeper/server1/data";
@@ -43,7 +46,7 @@ public class ZkSolrClientTest extends Ab
 
     server = new ZkTestServer(zkDir);
     server.run();
-
+    AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
     SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
 
     zkClient.close();
@@ -57,7 +60,7 @@ public class ZkSolrClientTest extends Ab
 
     server = new ZkTestServer(zkDir);
     server.run();
-
+    AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
     AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
 
     SolrZkClient zkClient = new SolrZkClient(server.getZkHost(),
@@ -103,7 +106,7 @@ public class ZkSolrClientTest extends Ab
     try {
       server = new ZkTestServer(zkDir);
       server.run();
-
+      AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
       AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
 
       zkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
@@ -196,6 +199,7 @@ public class ZkSolrClientTest extends Ab
     final AtomicInteger cnt = new AtomicInteger();
     ZkTestServer server = new ZkTestServer(zkDir);
     server.run();
+    AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
     Thread.sleep(400);
     AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
     final SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
@@ -261,16 +265,6 @@ public class ZkSolrClientTest extends Ab
   }
 
   @Override
-  public String getSchemaFile() {
-    return null;
-  }
-
-  @Override
-  public String getSolrConfigFile() {
-    return null;
-  }
-  
-  @Override
   public void tearDown() throws Exception {
     super.tearDown();
   }



Mime
View raw message