lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1423754 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ lucene/ lucene/analysis/ lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/backwards/ lucene/benchmark/ lucene/codecs/ lucene/core/ lucene/core/src/test/org/apache/l...
Date Wed, 19 Dec 2012 03:48:20 GMT
Author: yonik
Date: Wed Dec 19 03:48:19 2012
New Revision: 1423754

URL: http://svn.apache.org/viewvc?rev=1423754&view=rev
Log:
tests: start single jetty in cloud mode w/ manual shard assignment

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/BUILD.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/MIGRATE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/README.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_4x/lucene/benchmark/   (props changed)
    lucene/dev/branches/branch_4x/lucene/build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip
  (props changed)
    lucene/dev/branches/branch_4x/lucene/demo/   (props changed)
    lucene/dev/branches/branch_4x/lucene/facet/   (props changed)
    lucene/dev/branches/branch_4x/lucene/grouping/   (props changed)
    lucene/dev/branches/branch_4x/lucene/highlighter/   (props changed)
    lucene/dev/branches/branch_4x/lucene/ivy-settings.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/join/   (props changed)
    lucene/dev/branches/branch_4x/lucene/licenses/   (props changed)
    lucene/dev/branches/branch_4x/lucene/memory/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/module-build.xml   (props changed)
    lucene/dev/branches/branch_4x/lucene/queries/   (props changed)
    lucene/dev/branches/branch_4x/lucene/queryparser/   (props changed)
    lucene/dev/branches/branch_4x/lucene/sandbox/   (props changed)
    lucene/dev/branches/branch_4x/lucene/site/   (props changed)
    lucene/dev/branches/branch_4x/lucene/spatial/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/tools/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/LICENSE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/README.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/cloud-dev/   (props changed)
    lucene/dev/branches/branch_4x/solr/common-build.xml   (props changed)
    lucene/dev/branches/branch_4x/solr/contrib/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java
    lucene/dev/branches/branch_4x/solr/example/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt   (props changed)
    lucene/dev/branches/branch_4x/solr/scripts/   (props changed)
    lucene/dev/branches/branch_4x/solr/site/   (props changed)
    lucene/dev/branches/branch_4x/solr/solrj/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
    lucene/dev/branches/branch_4x/solr/testlogging.properties   (props changed)
    lucene/dev/branches/branch_4x/solr/webapp/   (props changed)

Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java?rev=1423754&r1=1423753&r2=1423754&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/ShardRoutingCustomTest.java
Wed Dec 19 03:48:19 2012
@@ -32,6 +32,8 @@ import java.util.Set;
 
 public class ShardRoutingCustomTest extends AbstractFullDistribZkTestBase {
 
+  String collection = DEFAULT_COLLECTION;  // enable this to be configurable (more work needs
to be done)
+
   @BeforeClass
   public static void beforeShardHashingTest() throws Exception {
     useFactory(null);
@@ -51,7 +53,7 @@ public class ShardRoutingCustomTest exte
       testFinished = true;
     } finally {
       if (!testFinished) {
-        printLayoutOnTearDown = true;
+        printLayout();
       }
     }
   }
@@ -59,8 +61,9 @@ public class ShardRoutingCustomTest exte
   private void doCustomSharding() throws Exception {
     printLayout();
 
+    startCloudJetty(collection, "shardA");
 
-
+    printLayout();
   }
 
 

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1423754&r1=1423753&r2=1423754&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
(original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Wed Dec 19 03:48:19 2012
@@ -259,6 +259,7 @@ public abstract class AbstractFullDistri
       controlClientCloud.connect();
       waitForCollection(controlClientCloud.getZkStateReader(), "control_collection", 0);
       // NOTE: we are skipping creation of the chaos monkey by returning here
+      cloudClient = controlClientCloud;  // temporary - some code needs/uses cloudClient
       return;
     }
 
@@ -326,10 +327,10 @@ public abstract class AbstractFullDistri
     if (checkCreatedVsState) {
       // now wait until we see that the number of shards in the cluster state
       // matches what we expect
-      int numShards = getNumShards(DEFAULT_COLLECTION);
+      int numShards = getTotalReplicas(DEFAULT_COLLECTION);
       int retries = 0;
       while (numShards != shardCount) {
-        numShards = getNumShards(DEFAULT_COLLECTION);
+        numShards = getTotalReplicas(DEFAULT_COLLECTION);
         if (numShards == shardCount) break;
         if (retries++ == 60) {
           printLayoutOnTearDown = true;
@@ -361,17 +362,53 @@ public abstract class AbstractFullDistri
     return jettys;
   }
 
-  protected int getNumShards(String collection) {
-    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
-    Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(collection);
-    if (slices == null) {
-      throw new IllegalArgumentException("Could not find collection:" + collection);
+
+  protected SolrServer startCloudJetty(String collection, String shard) throws Exception
{
+
+    // TODO: use the collection string!!!!
+    collection = DEFAULT_COLLECTION;
+
+    int totalReplicas = getTotalReplicas(collection);
+
+
+    int cnt = this.jettyIntCntr.incrementAndGet();
+      File jettyDir = new File(TEMP_DIR,
+          getClass().getName() + "-jetty" + cnt + "-" + System.currentTimeMillis());
+      jettyDir.mkdirs();
+      org.apache.commons.io.FileUtils.copyDirectory(new File(getSolrHome()), jettyDir);
+      JettySolrRunner j = createJetty(jettyDir, testDir + "/jetty" + cnt, shard, "solrconfig.xml",
null);
+      jettys.add(j);
+      SolrServer client = createNewSolrServer(j.getLocalPort());
+      clients.add(client);
+
+    int retries = 60;
+    while (--retries >= 0) {
+      // total replicas changed.. assume it was us
+      if (getTotalReplicas(collection) != totalReplicas) {
+       break;
+      }
+      Thread.sleep(500);
     }
+
+    if (retries <= 0) {
+      fail("Timeout waiting for " + j + " to appear in clusterstate");
+      printLayout();
+    }
+
+    updateMappingsFromZk(this.jettys, this.clients);
+    return client;
+  }
+
+
+  /* Total number of replicas (number of cores serving an index to the collection) shown
by the cluster state */
+  protected int getTotalReplicas(String collection) {
+    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
+    DocCollection coll = zkStateReader.getClusterState().getCollectionStates().get(collection);
+    if (coll == null) return 0;  // support for when collection hasn't been created yet
     int cnt = 0;
-    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
-      cnt += entry.getValue().getReplicasMap().size();
+    for (Slice slices : coll.getSlices()) {
+      cnt += slices.getReplicas().size();
     }
-    
     return cnt;
   }
   
@@ -842,24 +879,24 @@ public abstract class AbstractFullDistri
     
   }
   
-  private String toStr(SolrDocumentList lst) {
-    if (lst.size() <= 10) return lst.toString();
+  private String toStr(SolrDocumentList lst, int maxSz) {
+    if (lst.size() <= maxSz) return lst.toString();
 
     StringBuilder sb = new StringBuilder("SolrDocumentList[sz=" + lst.size());
     if (lst.size() != lst.getNumFound()) {
       sb.append(" numFound=" + lst.getNumFound());
     }
     sb.append("]=");
-    sb.append(lst.subList(0,5).toString());
+    sb.append(lst.subList(0,maxSz/2).toString());
     sb.append(" , [...] , ");
-    sb.append(lst.subList(lst.size()-5, lst.size()).toString());
+    sb.append(lst.subList(lst.size()-maxSz/2, lst.size()).toString());
 
     return sb.toString();
   }
 
   Set<Map> showDiff(SolrDocumentList a, SolrDocumentList b, String aName, String bName)
{
-    System.err.println("######"+aName+ ": " + toStr(a));
-    System.err.println("######"+bName+ ": " + toStr(b));
+    System.err.println("######"+aName+ ": " + toStr(a,10));
+    System.err.println("######"+bName+ ": " + toStr(b,10));
     System.err.println("###### sizes=" + a.size() + "," + b.size());
     
     Set<Map> setA = new HashSet<Map>();



Mime
View raw message