lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject svn commit: r1373412 - in /lucene/dev/branches/branch_4x: ./ solr/ solr/core/ solr/core/src/java/org/apache/solr/core/ solr/core/src/java/org/apache/solr/handler/ solr/core/src/java/org/apache/solr/update/ solr/core/src/test-files/solr/collection1/conf...
Date Wed, 15 Aug 2012 14:00:16 GMT
Author: markrmiller
Date: Wed Aug 15 14:00:14 2012
New Revision: 1373412

URL: http://svn.apache.org/viewvc?rev=1373412&view=rev
Log:
SOLR-3730: Rollback is not implemented quite right and can cause corner case fails in SolrCloud
tests.

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/solr/core/   (props changed)
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
    lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
    lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java

Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Aug 15 14:00:14 2012
@@ -59,6 +59,9 @@ Bug Fixes
 * SOLR-3649: Fixed bug in JavabinLoader that caused deleteById(List<String> ids)
   to not work in SolrJ (siren)
 
+* SOLR-3730: Rollback is not implemented quite right and can cause corner case fails in 
+  SolrCloud tests. (rmuir, Mark Miller)
+
 ==================  4.0.0-BETA ===================
 
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
Wed Aug 15 14:00:14 2012
@@ -125,6 +125,7 @@ public abstract class CachingDirectoryFa
       }
       cacheValue.refCnt--;
       if (cacheValue.refCnt == 0 && cacheValue.doneWithDir) {
+        log.info("Closing directory:" + cacheValue.path);
         directory.close();
         byDirectoryCache.remove(directory);
         byPathCache.remove(cacheValue.path);
@@ -194,6 +195,7 @@ public abstract class CachingDirectoryFa
         
         byDirectoryCache.put(directory, newCacheValue);
         byPathCache.put(fullPath, newCacheValue);
+        log.info("return new directory for " + fullPath + " forceNew:" + forceNew);
       } else {
         cacheValue.refCnt++;
       }

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Wed
Aug 15 14:00:14 2012
@@ -1554,7 +1554,7 @@ public final class SolrCore implements S
         } catch (Throwable e) {
           // do not allow decref() operations to fail since they are typically called in
finally blocks
           // and throwing another exception would be very unexpected.
-          SolrException.log(log, "Error closing searcher:", e);
+          SolrException.log(log, "Error closing searcher:" + this, e);
         }
       }
     };

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
Wed Aug 15 14:00:14 2012
@@ -384,7 +384,7 @@ public class SnapPuller {
             // may be closed
             core.getDirectoryFactory().doneWithDirectory(oldDirectory);
           }
-          doCommit();
+          doCommit(isFullCopyNeeded);
         }
         
         replicationStartTime = 0;
@@ -533,11 +533,11 @@ public class SnapPuller {
     return sb;
   }
 
-  private void doCommit() throws IOException {
+  private void doCommit(boolean isFullCopyNeeded) throws IOException {
     SolrQueryRequest req = new LocalSolrQueryRequest(solrCore,
         new ModifiableSolrParams());
     // reboot the writer on the new index and get a new searcher
-    solrCore.getUpdateHandler().newIndexWriter(true);
+    solrCore.getUpdateHandler().newIndexWriter(isFullCopyNeeded);
     
     try {
       // first try to open an NRT searcher so that the new 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
Wed Aug 15 14:00:14 2012
@@ -74,8 +74,7 @@ public final class DefaultSolrCoreState 
       }
       
       if (indexWriter == null) {
-        indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
-            false, false);
+        indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", false);
       }
       if (refCntWriter == null) {
         refCntWriter = new RefCounted<IndexWriter>(indexWriter) {
@@ -110,18 +109,28 @@ public final class DefaultSolrCoreState 
           writerPauseLock.wait();
         } catch (InterruptedException e) {}
       }
-      
+
       try {
         if (indexWriter != null) {
-          try {
-            log.info("Closing old IndexWriter... core=" + coreName);
-            indexWriter.close();
-          } catch (Throwable t) {
-            SolrException.log(log, "Error closing old IndexWriter. core=" + coreName, t);
+          if (!rollback) {
+            try {
+              log.info("Closing old IndexWriter... core=" + coreName);
+              indexWriter.close();
+            } catch (Throwable t) {
+              SolrException.log(log, "Error closing old IndexWriter. core="
+                  + coreName, t);
+            }
+          } else {
+            try {
+              log.info("Rollback old IndexWriter... core=" + coreName);
+              indexWriter.rollback();
+            } catch (Throwable t) {
+              SolrException.log(log, "Error rolling back old IndexWriter. core="
+                  + coreName, t);
+            }
           }
         }
-        indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2",
-            false, true);
+        indexWriter = createMainIndexWriter(core, "DirectUpdateHandler2", true);
         log.info("New IndexWriter is ready to be used.");
         // we need to null this so it picks up the new writer next get call
         refCntWriter = null;
@@ -174,14 +183,12 @@ public final class DefaultSolrCoreState 
 
   @Override
   public synchronized void rollbackIndexWriter(SolrCore core) throws IOException {
-    indexWriter.rollback();
     newIndexWriter(core, true);
   }
   
-  protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name,
-      boolean removeAllExisting, boolean forceNewDirectory) throws IOException {
+  protected SolrIndexWriter createMainIndexWriter(SolrCore core, String name, boolean forceNewDirectory)
throws IOException {
     return new SolrIndexWriter(name, core.getNewIndexDir(),
-        core.getDirectoryFactory(), removeAllExisting, core.getSchema(),
+        core.getDirectoryFactory(), false, core.getSchema(),
         core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory);
   }
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
Wed Aug 15 14:00:14 2012
@@ -141,6 +141,8 @@ public class SolrIndexWriter extends Ind
       super.rollback();
     } finally {
       isClosed = true;
+      directoryFactory.release(getDirectory());
+      numCloses.incrementAndGet();
     }
   }
 

Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
(original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml
Wed Aug 15 14:00:14 2012
@@ -54,7 +54,7 @@
     -->
     <maxBufferedDocs>10</maxBufferedDocs>
     <mergePolicy class="org.apache.lucene.index.LogDocMergePolicy"/>
-    <lockType>single</lockType>
+    <lockType>native</lockType>
     <unlockOnStartup>true</unlockOnStartup>
   </indexConfig>
   

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
Wed Aug 15 14:00:14 2012
@@ -32,10 +32,12 @@ public class MockDirectoryFactory extend
   @Override
   protected Directory create(String path) throws IOException {
     Directory dir = LuceneTestCase.newDirectory();
-    // Somehow removing unref'd files in Solr tests causes
-    // problems... there's some interaction w/
-    // CachingDirectoryFactory.  Once we track down where Solr
-    // isn't closing an IW, we can re-enable this:
+    // we can't currently do this check because of how
+    // Solr has to reboot a new Directory sometimes when replicating
+    // or rolling back - the old directory is closed and the following
+    // test assumes it can open an IndexWriter when that happens - we
+    // have a new Directory for the same dir and still an open IW at 
+    // this point
     if (dir instanceof MockDirectoryWrapper) {
       ((MockDirectoryWrapper)dir).setAssertNoUnrefencedFilesOnClose(false);
     }

Modified: lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java?rev=1373412&r1=1373411&r2=1373412&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java
Wed Aug 15 14:00:14 2012
@@ -32,10 +32,12 @@ public class MockFSDirectoryFactory exte
   @Override
   public Directory create(String path) throws IOException {
     Directory dir = LuceneTestCase.newFSDirectory(new File(path));
-    // Somehow removing unref'd files in Solr tests causes
-    // problems... there's some interaction w/
-    // CachingDirectoryFactory.  Once we track down where Solr
-    // isn't closing an IW, we can re-enable this:
+    // we can't currently do this check because of how
+    // Solr has to reboot a new Directory sometimes when replicating
+    // or rolling back - the old directory is closed and the following
+    // test assumes it can open an IndexWriter when that happens - we
+    // have a new Directory for the same dir and still an open IW at 
+    // this point
     if (dir instanceof MockDirectoryWrapper) {
       ((MockDirectoryWrapper)dir).setAssertNoUnrefencedFilesOnClose(false);
     }



Mime
View raw message