lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1207595 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/src/java/org/apache/lucene/index/ lucene/src/test-framework/java/org/apache/lucene/store/ lucene/src/test/org/apache/lucene/index/ solr/ solr/test-framework/src/java/org/apache...
Date Mon, 28 Nov 2011 20:53:41 GMT
Author: mikemccand
Date: Mon Nov 28 20:53:40 2011
New Revision: 1207595

URL: http://svn.apache.org/viewvc?rev=1207595&view=rev
Log:
LUCENE-3601: assert no unref'd files on MDW.close; fix IW to track explicit files when incRefing
files during commit

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/branch_3x/solr/   (props changed)
    lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1207595&r1=1207594&r2=1207595&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
Mon Nov 28 20:53:40 2011
@@ -31,27 +31,27 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
-import org.apache.lucene.search.Similarity;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.BufferedIndexInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.MapBackedSet;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.ThreadInterruptedException;
-import org.apache.lucene.util.Version;
-import org.apache.lucene.util.MapBackedSet;
 import org.apache.lucene.util.TwoPhaseCommit;
+import org.apache.lucene.util.Version;
 
 /**
   An <code>IndexWriter</code> creates and maintains an index.
@@ -277,6 +277,7 @@ public class IndexWriter implements Clos
   volatile long pendingCommitChangeCount;
 
   final SegmentInfos segmentInfos = new SegmentInfos();       // the segments
+  private Collection<String> filesToCommit;
 
   private DocumentsWriter docWriter;
   private IndexFileDeleter deleter;
@@ -3381,7 +3382,8 @@ public class IndexWriter implements Clos
           // we are trying to sync all referenced files, a
           // merge completes which would otherwise have
           // removed the files we are now syncing.
-          deleter.incRef(toCommit, false);
+          filesToCommit = toCommit.files(directory, false);
+          deleter.incRef(filesToCommit);
         }
         success = true;
       } finally {
@@ -3403,7 +3405,8 @@ public class IndexWriter implements Clos
     } finally {
       if (!success) {
         synchronized (this) {
-          deleter.decRef(toCommit);
+          deleter.decRef(filesToCommit);
+          filesToCommit = null;
         }
       }
     }
@@ -3503,8 +3506,9 @@ public class IndexWriter implements Clos
         rollbackSegments = pendingCommit.createBackupSegmentInfos(true);
         deleter.checkpoint(pendingCommit, true);
       } finally {
-        // Matches the incRef done in startCommit:
-        deleter.decRef(pendingCommit);
+        // Matches the incRef done in prepareCommit:
+        deleter.decRef(filesToCommit);
+        filesToCommit = null;
         pendingCommit = null;
         notifyAll();
       }
@@ -4518,7 +4522,8 @@ public class IndexWriter implements Clos
           if (infoStream != null) {
             message("  skip startCommit(): no changes pending");
           }
-          deleter.decRef(toSync);
+          deleter.decRef(filesToCommit);
+          filesToCommit = null;
           return;
         }
         
@@ -4581,7 +4586,8 @@ public class IndexWriter implements Clos
               message("hit exception committing segments file");
             }
 
-            deleter.decRef(toSync);
+            deleter.decRef(filesToCommit);
+            filesToCommit = null;
           }
         }
       }

Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1207595&r1=1207594&r2=1207595&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
Mon Nov 28 20:53:40 2011
@@ -34,6 +34,8 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.ThrottledIndexOutput;
 import org.apache.lucene.util._TestUtil;
@@ -497,6 +499,12 @@ public class MockDirectoryWrapper extend
     return size;
   }
 
+  private boolean assertNoUnreferencedFilesOnClose = true;
+
+  public void setAssertNoUnrefencedFilesOnClose(boolean v) {
+    assertNoUnreferencedFilesOnClose = v;
+  }
+
   @Override
   public synchronized void close() throws IOException {
     maybeYield();
@@ -529,6 +537,20 @@ public class MockDirectoryWrapper extend
           System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
         } 
         _TestUtil.checkIndex(this);
+
+        if (assertNoUnreferencedFilesOnClose) {
+          // now look for unreferenced files:
+          String[] startFiles = listAll();
+          new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT,
null)).rollback();
+          String[] endFiles = listAll();
+
+          Arrays.sort(startFiles);
+          Arrays.sort(endFiles);
+
+          if (!Arrays.equals(startFiles, endFiles)) {
+            assert false : "unreferenced files: before delete:\n    " + Arrays.toString(startFiles)
+ "\n  after delete:\n    " + Arrays.toString(endFiles);
+          }
+        }
       }
     }
     delegate.close();

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1207595&r1=1207594&r2=1207595&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
(original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
Mon Nov 28 20:53:40 2011
@@ -143,8 +143,6 @@ public class TestIndexWriter extends Luc
 
     public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException
{
       String[] startFiles = dir.listAll();
-      SegmentInfos infos = new SegmentInfos();
-      infos.read(dir);
       new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))).rollback();
       String[] endFiles = dir.listAll();
 

Modified: lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java?rev=1207595&r1=1207594&r2=1207595&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
(original)
+++ lucene/dev/branches/branch_3x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
Mon Nov 28 20:53:40 2011
@@ -21,6 +21,7 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 
 /**
@@ -30,6 +31,12 @@ public class MockDirectoryFactory extend
 
   @Override
   public Directory open(String path) throws IOException {
-    return LuceneTestCase.newFSDirectory(new File(path));
+    MockDirectoryWrapper dir = LuceneTestCase.newFSDirectory(new File(path));
+    // Somehow removing unref'd files in Solr tests causes
+    // problems... there's some interaction w/
+    // CachingDirectoryFactory.  Once we track down where Solr
+    // isn't closing an IW, we can re-enable this:
+    dir.setAssertNoUnrefencedFilesOnClose(false);
+    return dir;
   }
 }



Mime
View raw message