lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yo...@apache.org
Subject svn commit: r1423591 - in /lucene/dev/trunk/solr: core/src/test/org/apache/solr/cloud/ test-framework/src/java/org/apache/solr/cloud/
Date Tue, 18 Dec 2012 19:30:40 GMT
Author: yonik
Date: Tue Dec 18 19:30:39 2012
New Revision: 1423591

URL: http://svn.apache.org/viewvc?rev=1423591&view=rev
Log:
SOLR-3180: improve logging

Modified:
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1423591&r1=1423590&r2=1423591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
(original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
Tue Dec 18 19:30:39 2012
@@ -49,6 +49,8 @@ public class ChaosMonkeySafeLeaderTest e
   @Before
   @Override
   public void setUp() throws Exception {
+    useFactory("solr.StandardDirectoryFactory");
+
     super.setUp();
     
     System.setProperty("numShards", Integer.toString(sliceCount));

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1423591&r1=1423590&r2=1423591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Tue Dec
18 19:30:39 2012
@@ -89,7 +89,7 @@ public class RecoveryZkTest extends Abst
 
     // test that leader and replica have same doc count
     
-    checkShardConsistency("shard1", false); 
+    checkShardConsistency("shard1", false, false);
     SolrQuery query = new SolrQuery("*:*");
     query.setParam("distrib", "false");
     long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();

Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1423591&r1=1423590&r2=1423591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Tue Dec 18
19:30:39 2012
@@ -114,7 +114,7 @@ public class SyncSliceTest extends Abstr
     waitForRecoveriesToFinish(false);
 
     // shard should be inconsistent
-    String shardFailMessage = checkShardConsistency("shard1", true);
+    String shardFailMessage = checkShardConsistency("shard1", true, false);
     assertNotNull(shardFailMessage);
     
     ModifiableSolrParams params = new ModifiableSolrParams();
@@ -209,7 +209,7 @@ public class SyncSliceTest extends Abstr
     waitForRecoveriesToFinish(false);
     
     // shard should be inconsistent
-    shardFailMessage = checkShardConsistency("shard1", true);
+    shardFailMessage = checkShardConsistency("shard1", true, false);
     assertNotNull(
         "shard1 shoul have just been set up to be inconsistent - but it's still consistent",
         shardFailMessage); 

Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1423591&r1=1423590&r2=1423591&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
(original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
Tue Dec 18 19:30:39 2012
@@ -740,10 +740,14 @@ public abstract class AbstractFullDistri
   }
   
   protected void checkShardConsistency(String shard) throws Exception {
-    checkShardConsistency(shard, false);
+    checkShardConsistency(shard, false, false);
   }
-  
-  protected String checkShardConsistency(String shard, boolean verbose)
+
+  /* Returns a non-null string if replicas within the same shard are not consistent.
+   * If expectFailure==false, the exact differences found will be logged since this would
be an unexpected failure.
+   * verbose causes extra debugging into to be displayed, even if everything is consistent.
+   */
+  protected String checkShardConsistency(String shard, boolean expectFailure, boolean verbose)
       throws Exception {
     
     List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
@@ -768,10 +772,9 @@ public abstract class AbstractFullDistri
       ZkNodeProps props = cjetty.info;
       if (verbose) System.err.println("client" + cnt++);
       if (verbose) System.err.println("PROPS:" + props);
-      
+
       try {
-        SolrQuery query = new SolrQuery("*:*");
-        query.set("distrib", false);
+        SolrParams query = params("q","*:*", "rows","0", "distrib","false", "tests","checkShardConsistency");
// "tests" is just a tag that won't do anything except be echoed in logs
         num = cjetty.client.solrClient.query(query).getResults().getNumFound();
       } catch (SolrServerException e) {
         if (verbose) System.err.println("error contacting client: "
@@ -789,9 +792,8 @@ public abstract class AbstractFullDistri
         live = true;
       }
       if (verbose) System.err.println(" live:" + live);
-      
       if (verbose) System.err.println(" num:" + num + "\n");
-      
+
       boolean active = props.getStr(ZkStateReader.STATE_PROP).equals(
           ZkStateReader.ACTIVE);
       if (active && live) {
@@ -799,13 +801,14 @@ public abstract class AbstractFullDistri
           failMessage = shard + " is not consistent.  Got " + lastNum + " from " + lastJetty.url
+ "lastClient"
               + " and got " + num + " from " + cjetty.url;
 
-          if (verbose || true) {
+          if (!expectFailure || verbose) {
             System.err.println("######" + failMessage);
             SolrQuery query = new SolrQuery("*:*");
             query.set("distrib", false);
             query.set("fl","id,_version_");
-            query.set("rows","1000");
+            query.set("rows","100000");
             query.set("sort","id asc");
+            query.set("tests","checkShardConsistency/showDiff");
 
             SolrDocumentList lst1 = lastJetty.client.solrClient.query(query).getResults();
             SolrDocumentList lst2 = cjetty.client.solrClient.query(query).getResults();
@@ -822,9 +825,24 @@ public abstract class AbstractFullDistri
     
   }
   
-  void showDiff(SolrDocumentList a, SolrDocumentList b, String aName, String bName) {
-    System.err.println("######"+aName+ ": " + a);
-    System.err.println("######"+bName+ ": " + b);
+  private String toStr(SolrDocumentList lst) {
+    if (lst.size() <= 10) return lst.toString();
+
+    StringBuilder sb = new StringBuilder("SolrDocumentList[sz=" + lst.size());
+    if (lst.size() != lst.getNumFound()) {
+      sb.append(" numFound=" + lst.getNumFound());
+    }
+    sb.append("]=");
+    sb.append(lst.subList(0,5).toString());
+    sb.append(" , [...] , ");
+    sb.append(lst.subList(lst.size()-5, lst.size()).toString());
+
+    return sb.toString();
+  }
+
+  Set<Map> showDiff(SolrDocumentList a, SolrDocumentList b, String aName, String bName)
{
+    System.err.println("######"+aName+ ": " + toStr(a));
+    System.err.println("######"+bName+ ": " + toStr(b));
     System.err.println("###### sizes=" + a.size() + "," + b.size());
     
     Set<Map> setA = new HashSet<Map>();
@@ -848,30 +866,30 @@ public abstract class AbstractFullDistri
     if (onlyInB.size() > 0) {
       System.err.println("###### Only in " + bName + ": " + onlyInB);
     }
+
+    onlyInA.addAll(b);
+    return onlyInA;
   }
-  
+
+  /* Checks both shard replcia consistency and against the control shard.
+  * The test will be failed if differences are found.
+  */
   protected void checkShardConsistency() throws Exception {
     checkShardConsistency(true, false);
   }
-  
+
+  /* Checks shard consistency and optionally checks against the control shard.
+   * The test will be failed if differences are found.
+   */
   protected void checkShardConsistency(boolean checkVsControl, boolean verbose)
       throws Exception {
-    SolrParams q = params("q","*:*","fl","id","rows","100000");
-
-    SolrDocumentList controlDocList = controlClient.query(q).getResults();
-    long docs = controlDocList.getNumFound();
-
-    SolrDocumentList cloudDocList = cloudClient.query(q).getResults();
-    long cloudClientDocs = cloudDocList.getNumFound();
 
-    if (verbose) System.err.println("Control Docs:" + docs);
-    
     updateMappingsFromZk(jettys, clients);
     
     Set<String> theShards = shardToJetty.keySet();
     String failMessage = null;
     for (String shard : theShards) {
-      String shardFailMessage = checkShardConsistency(shard, verbose);
+      String shardFailMessage = checkShardConsistency(shard, false, verbose);
       if (shardFailMessage != null && failMessage == null) {
         failMessage = shardFailMessage;
       }
@@ -880,47 +898,88 @@ public abstract class AbstractFullDistri
     if (failMessage != null) {
       fail(failMessage);
     }
-    
-    if (checkVsControl) {
-      // now check that the right # are on each shard
-      theShards = shardToJetty.keySet();
-      int cnt = 0;
-      for (String s : theShards) {
-        int times = shardToJetty.get(s).size();
-        for (int i = 0; i < times; i++) {
-          try {
-            CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
-            ZkNodeProps props = cjetty.info;
-            SolrServer client = cjetty.client.solrClient;
-            boolean active = props.getStr(ZkStateReader.STATE_PROP).equals(
-                ZkStateReader.ACTIVE);
-            if (active) {
-              SolrQuery query = new SolrQuery("*:*");
-              query.set("distrib", false);
-              long results = client.query(query).getResults().getNumFound();
-              if (verbose) System.err.println(new ZkCoreNodeProps(props)
-                  .getCoreUrl() + " : " + results);
-              if (verbose) System.err.println("shard:"
-                  + props.getStr(ZkStateReader.SHARD_ID_PROP));
-              cnt += results;
-              break;
-            }
-          } catch (Exception e) {
-            // if we have a problem, try the next one
-            if (i == times - 1) {
-              throw e;
-            }
+
+    if (!checkVsControl) return;
+
+    SolrParams q = params("q","*:*","rows","0", "tests","checkShardConsistency(vsControl)");
   // add a tag to aid in debugging via logs
+
+    SolrDocumentList controlDocList = controlClient.query(q).getResults();
+    long controlDocs = controlDocList.getNumFound();
+
+    SolrDocumentList cloudDocList = cloudClient.query(q).getResults();
+    long cloudClientDocs = cloudDocList.getNumFound();
+
+
+
+
+    // now check that the right # are on each shard
+    theShards = shardToJetty.keySet();
+    int cnt = 0;
+    for (String s : theShards) {
+      int times = shardToJetty.get(s).size();
+      for (int i = 0; i < times; i++) {
+        try {
+          CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
+          ZkNodeProps props = cjetty.info;
+          SolrServer client = cjetty.client.solrClient;
+          boolean active = props.getStr(ZkStateReader.STATE_PROP).equals(
+              ZkStateReader.ACTIVE);
+          if (active) {
+            SolrQuery query = new SolrQuery("*:*");
+            query.set("distrib", false);
+            long results = client.query(query).getResults().getNumFound();
+            if (verbose) System.err.println(new ZkCoreNodeProps(props)
+                .getCoreUrl() + " : " + results);
+            if (verbose) System.err.println("shard:"
+                + props.getStr(ZkStateReader.SHARD_ID_PROP));
+            cnt += results;
+            break;
+          }
+        } catch (Exception e) {
+          // if we have a problem, try the next one
+          if (i == times - 1) {
+            throw e;
           }
         }
       }
-      
+    }
+
+
+    if (controlDocs != cnt || cloudClientDocs != controlDocs) {
+      String msg = "document count mismatch.  control=" + controlDocs + " sum(shards)="+
cnt + " cloudClient="+cloudClientDocs;
+      log.error(msg);
 
-      if (docs != cnt || cloudClientDocs != docs) {
-        String msg = "document count mismatch.  control=" + docs + " sum(shards)="+ cnt +
" cloudClient="+cloudClientDocs;
-        log.error(msg);
-        showDiff(controlDocList, cloudDocList,"controlDocList","cloudDocList");
-        fail(msg);
+      // re-execute the query getting ids
+      q = params("q","*:*","rows","100000", "fl","id", "tests","checkShardConsistency(vsControl)/getIds");
   // add a tag to aid in debugging via logs
+      controlDocList = controlClient.query(q).getResults();
+      if (controlDocs != controlDocList.getNumFound()) {
+        log.error("Something changed! control now " + controlDocList.getNumFound());
+      };
+
+      cloudDocList = cloudClient.query(q).getResults();
+      if (cloudClientDocs != cloudDocList.getNumFound()) {
+        log.error("Something changed! cloudClient now " + cloudDocList.getNumFound());
+      };
+
+      Set<Map> differences = showDiff(controlDocList, cloudDocList,"controlDocList","cloudDocList");
+
+      // get versions for the mismatched ids
+      StringBuilder ids = new StringBuilder("id:(");
+      for (Map doc : differences) {
+        ids.append(" "+doc.get("id"));
       }
+      ids.append(")");
+
+      // get versions for those ids that don't match
+      q = params("q",ids.toString(),"rows","100000", "fl","id,_version_", "sort","id asc",
+                 "tests","checkShardConsistency(vsControl)/getVers");    // add a tag to
aid in debugging via logs
+
+      SolrDocumentList a = controlClient.query(q).getResults();
+      SolrDocumentList b = cloudClient.query(q).getResults();
+
+      log.error("controlClient :" + a + "\n\tcloudClient :" + b);
+
+      fail(msg);
     }
   }
   
@@ -1159,7 +1218,7 @@ public abstract class AbstractFullDistri
       Set<String> theShards = shardToJetty.keySet();
       String failMessage = null;
       for (String shard : theShards) {
-        failMessage = checkShardConsistency(shard, false);
+        failMessage = checkShardConsistency(shard, true, false);
       }
       
       if (failMessage != null) {



Mime
View raw message