lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1393408 - in /lucene/dev/trunk/lucene: ./ codecs/src/java/org/apache/lucene/codecs/simpletext/ core/src/java/org/apache/lucene/index/ core/src/java/org/apache/lucene/store/ core/src/test/org/apache/lucene/codecs/perfield/ core/src/test/org...
Date Wed, 03 Oct 2012 11:01:15 GMT
Author: mikemccand
Date: Wed Oct  3 11:01:13 2012
New Revision: 1393408

URL: http://svn.apache.org/viewvc?rev=1393408&view=rev
Log:
LUCENE-4456: fix various bugs (see lucene/CHANGES.txt)

Added:
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java   (with props)
Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
    lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
    lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed Oct  3 11:01:13 2012
@@ -226,6 +226,14 @@ Bug Fixes
   CheckIndex that would report no deletions when a segment has
   deletions, and vice/versa.  (Uwe Schindler, Robert Muir, Mike McCandless)
 
+* LUCENE-4456: Fixed double-counting sizeInBytes for a segment
+  (affects how merge policies pick merges); fixed CheckIndex's
+  incorrect reporting of whether a segment has deletions; fixed case
+  where on abort Lucene could remove files it didn't create; fixed
+  many cases where IndexWriter could leave leftover files (on
+  exception in various places, on reuse of a segment name after crash
+  and recovery.  (Uwe Schindler, Robert Muir, Mike McCandless)
+
 Optimizations
 
 * LUCENE-4322: Decrease lucene-core JAR size. The core JAR size had increased a

Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java Wed Oct  3 11:01:13 2012
@@ -198,6 +198,7 @@ public class SimpleTextDocValuesConsumer
         IOUtils.close(output);
       } else {
         IOUtils.closeWhileHandlingException(output);
+        dir.deleteFile(fileName);
       }
     }
   }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Wed Oct  3 11:01:13 2012
@@ -137,6 +137,7 @@ class DocumentsWriterPerThread {
    *  currently buffered docs.  This resets our state,
    *  discarding any docs added since last flush. */
   void abort() {
+    //System.out.println(Thread.currentThread().getName() + ": now abort seg=" + segmentInfo.name);
     hasAborted = aborting = true;
     try {
       if (infoStream.isEnabled("DWPT")) {

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Wed Oct  3 11:01:13 2012
@@ -333,12 +333,16 @@ final class IndexFileDeleter {
       segmentPrefix2 = null;
     }
 
+    Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
+      m.reset(fileName);
       if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
           !fileName.endsWith("write.lock") &&
           !refCounts.containsKey(fileName) &&
-          !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+          !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
+          (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
         // Unreferenced file, so remove it
         if (infoStream.isEnabled("IFD")) {
           infoStream.message("IFD", "refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
@@ -554,7 +558,13 @@ final class IndexFileDeleter {
   void deleteNewFiles(Collection<String> files) throws IOException {
     assert locked();
     for (final String fileName: files) {
-      if (!refCounts.containsKey(fileName)) {
+      // NOTE: it's very unusual yet possible for the
+      // refCount to be present and 0: it can happen if you
+      // open IW on a crashed index, and it removes a bunch
+      // of unref'd files, and then you add new docs / do
+      // merging, and it reuses that segment name.
+      // TestCrash.testCrashAfterReopen can hit this:
+      if (!refCounts.containsKey(fileName) || refCounts.get(fileName).count == 0) {
         if (infoStream.isEnabled("IFD")) {
           infoStream.message("IFD", "delete new file \"" + fileName + "\"");
         }

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Wed Oct  3 11:01:13 2012
@@ -2145,7 +2145,7 @@ public class IndexWriter implements Clos
         // Now build compound file
         Collection<String> oldFiles = createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, newSegment.info, context);
         newSegment.info.setUseCompoundFile(true);
-        
+
         synchronized(this) {
           deleter.deleteNewFiles(oldFiles);
         }
@@ -2321,31 +2321,62 @@ public class IndexWriter implements Clos
       flush(false, true);
 
       List<SegmentInfoPerCommit> infos = new ArrayList<SegmentInfoPerCommit>();
-      for (Directory dir : dirs) {
-        if (infoStream.isEnabled("IW")) {
-          infoStream.message("IW", "addIndexes: process directory " + dir);
-        }
-        SegmentInfos sis = new SegmentInfos(); // read infos from dir
-        sis.read(dir);
-
-        for (SegmentInfoPerCommit info : sis) {
-          assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
-
-          String newSegName = newSegmentName();
-          String dsName = info.info.name;
 
+      boolean success = false;
+      try {
+        for (Directory dir : dirs) {
           if (infoStream.isEnabled("IW")) {
-            infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+            infoStream.message("IW", "addIndexes: process directory " + dir);
           }
+          SegmentInfos sis = new SegmentInfos(); // read infos from dir
+          sis.read(dir);
+
+          for (SegmentInfoPerCommit info : sis) {
+            assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
+
+            String newSegName = newSegmentName();
+            String dsName = info.info.name;
 
-          IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
+            if (infoStream.isEnabled("IW")) {
+              infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+            }
+
+            IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
           
-          infos.add(copySegmentAsIs(info, newSegName, context));
+            infos.add(copySegmentAsIs(info, newSegName, context));
+          }
+        }
+        success = true;
+      } finally {
+        if (!success) {
+          for(SegmentInfoPerCommit sipc : infos) {
+            for(String file : sipc.files()) {
+              try {
+                directory.deleteFile(file);
+              } catch (Throwable t) {
+              }
+            }
+          }
         }
       }
 
       synchronized (this) {
-        ensureOpen();
+        success = false;
+        try {
+          ensureOpen();
+          success = true;
+        } finally {
+          if (!success) {
+            for(SegmentInfoPerCommit sipc : infos) {
+              for(String file : sipc.files()) {
+                try {
+                  directory.deleteFile(file);
+                } catch (Throwable t) {
+                }
+              }
+            }
+          }
+        }
         segmentInfos.addAll(infos);
         checkpoint();
       }
@@ -2420,7 +2451,18 @@ public class IndexWriter implements Clos
         merger.add(reader);
       }
 
-      MergeState mergeState = merger.merge();                // merge 'em
+      MergeState mergeState;
+      boolean success = false;
+      try {
+        mergeState = merger.merge();                // merge 'em
+        success = true;
+      } finally {
+        if (!success) { 
+          synchronized(this) {
+            deleter.refresh(info.name);
+          }
+        }
+      }
 
       SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, 0, -1L);
 
@@ -2442,12 +2484,14 @@ public class IndexWriter implements Clos
       // Now create the compound file if needed
       if (useCompoundFile) {
         Collection<String> filesToDelete = infoPerCommit.files();
-        createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
-
-        // delete new non cfs files directly: they were never
-        // registered with IFD
-        synchronized(this) {
-          deleter.deleteNewFiles(filesToDelete);
+        try{
+          createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
+        } finally {
+          // delete new non cfs files directly: they were never
+          // registered with IFD
+          synchronized(this) {
+            deleter.deleteNewFiles(filesToDelete);
+          }
         }
         info.setUseCompoundFile(true);
       }
@@ -2456,7 +2500,18 @@ public class IndexWriter implements Clos
       // creating CFS so that 1) .si isn't slurped into CFS,
       // and 2) .si reflects useCompoundFile=true change
       // above:
-      codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
+      success = false;
+      try {
+        codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
+        success = true;
+      } finally {
+        if (!success) {
+          synchronized(this) {
+            deleter.refresh(info.name);
+          }
+        }
+      }
+
       info.addFiles(trackingDir.getCreatedFiles());
 
       // Register the new segment
@@ -2512,23 +2567,38 @@ public class IndexWriter implements Clos
     // We must rewrite the SI file because it references segment name in its list of files, etc
     TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
 
-    newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
+    boolean success = false;
 
-    final Collection<String> siFiles = trackingDir.getCreatedFiles();
+    try {
 
-    // Copy the segment's files
-    for (String file: info.files()) {
+      newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
 
-      final String newFileName = segName + IndexFileNames.stripSegmentName(file);
+      final Collection<String> siFiles = trackingDir.getCreatedFiles();
 
-      if (siFiles.contains(newFileName)) {
-        // We already rewrote this above
-        continue;
-      }
+      // Copy the segment's files
+      for (String file: info.files()) {
+
+        final String newFileName = segName + IndexFileNames.stripSegmentName(file);
 
-      assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
+        if (siFiles.contains(newFileName)) {
+          // We already rewrote this above
+          continue;
+        }
 
-      info.info.dir.copy(directory, file, newFileName, context);
+        assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
+
+        info.info.dir.copy(directory, file, newFileName, context);
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        for(String file : newInfo.files()) {
+          try {
+            directory.deleteFile(file);
+          } catch (Throwable t) {
+          }
+        }
+      }
     }
     
     return newInfoPerCommit;
@@ -3091,6 +3161,7 @@ public class IndexWriter implements Clos
       if (infoStream.isEnabled("IW")) {
         infoStream.message("IW", "commitMerge: skip: it was aborted");
       }
+      deleter.deleteNewFiles(merge.info.files());
       return false;
     }
 
@@ -3136,6 +3207,11 @@ public class IndexWriter implements Clos
       }
     }
 
+    if (dropSegment) {
+      assert !segmentInfos.contains(merge.info);
+      deleter.deleteNewFiles(merge.info.files());
+    }
+
     // Must close before checkpoint, otherwise IFD won't be
     // able to delete the held-open files from the merge
     // readers:
@@ -3407,7 +3483,7 @@ public class IndexWriter implements Clos
     setDiagnostics(si, "merge", details);
 
     if (infoStream.isEnabled("IW")) {
-      infoStream.message("IW", "merge seg=" + merge.info.info.name);
+      infoStream.message("IW", "merge seg=" + merge.info.info.name + " " + segString(merge.segments));
     }
 
     assert merge.estimatedMergeBytes == 0;
@@ -4116,7 +4192,22 @@ public class IndexWriter implements Clos
     } catch(IOException ex) {
       prior = ex;
     } finally {
-      IOUtils.closeWhileHandlingException(prior, cfsDir);
+      boolean success = false;
+      try {
+        IOUtils.closeWhileHandlingException(prior, cfsDir);
+        success = true;
+      } finally {
+        if (!success) {
+          try {
+            directory.deleteFile(fileName);
+          } catch (Throwable t) {
+          }
+          try {
+            directory.deleteFile(IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+          } catch (Throwable t) {
+          }
+        }
+      }
     }
 
     // Replace all previous files with the CFS/CFE files:

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Wed Oct  3 11:01:13 2012
@@ -547,7 +547,7 @@ public class LiveIndexWriterConfig {
     sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n");
     sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n");
     sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms()).append("\n");
-    sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append("\n");
+    sb.append("mergedSegmentWarmer=").append(getMergedSegmentWarmer()).append("\n");
     sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()).append("\n");
     sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n"); // TODO: this should be private to the codec, not settable here
     sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName()).append("\n");

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java Wed Oct  3 11:01:13 2012
@@ -79,6 +79,7 @@ public class SegmentInfoPerCommit {
 
   /** Returns all files in use by this segment. */
   public Collection<String> files() throws IOException {
+    // Start from the wrapped info's files:
     Collection<String> files = new HashSet<String>(info.files());
 
     // Must separately add any live docs files:

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java Wed Oct  3 11:01:13 2012
@@ -201,7 +201,18 @@ public abstract class Directory implemen
     } catch (IOException ioe) {
       priorException = ioe;
     } finally {
-      IOUtils.closeWhileHandlingException(priorException, os, is);
+      boolean success = false;
+      try {
+        IOUtils.closeWhileHandlingException(priorException, os, is);
+        success = true;
+      } finally {
+        if (!success) {
+          try {
+            to.deleteFile(dest);
+          } catch (Throwable t) {
+          }
+        }
+      }
     }
   }
 

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java Wed Oct  3 11:01:13 2012
@@ -33,7 +33,7 @@ public class TestPerFieldPostingsFormat 
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    codec = new RandomCodec(new Random(random().nextLong()), Collections.EMPTY_SET);
+    codec = new RandomCodec(new Random(random().nextLong()), Collections.<String>emptySet());
   }
   
   @Override

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCrash.java Wed Oct  3 11:01:13 2012
@@ -20,12 +20,13 @@ package org.apache.lucene.index;
 import java.io.IOException;
 import java.util.Random;
 
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NoLockFactory;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.util.LuceneTestCase;
 
 public class TestCrash extends LuceneTestCase {
 
@@ -67,20 +68,42 @@ public class TestCrash extends LuceneTes
     // before any documents were added.
     IndexWriter writer = initIndex(random(), true);
     MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+    // We create leftover files because merging could be
+    // running when we crash:
+    dir.setAssertNoUnrefencedFilesOnClose(false);
+
     crash(writer);
+
     IndexReader reader = DirectoryReader.open(dir);
     assertTrue(reader.numDocs() < 157);
     reader.close();
+
+    // Make a new dir, copying from the crashed dir, and
+    // open IW on it, to confirm IW "recovers" after a
+    // crash:
+    Directory dir2 = newDirectory(dir);
     dir.close();
+
+    new RandomIndexWriter(random(), dir2).close();
+    dir2.close();
   }
 
   public void testWriterAfterCrash() throws IOException {
     // This test relies on being able to open a reader before any commit
     // happened, so we must create an initial commit just to allow that, but
     // before any documents were added.
+    System.out.println("TEST: initIndex");
     IndexWriter writer = initIndex(random(), true);
+    System.out.println("TEST: done initIndex");
     MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+    // We create leftover files because merging could be
+    // running / store files could be open when we crash:
+    dir.setAssertNoUnrefencedFilesOnClose(false);
+
     dir.setPreventDoubleWrite(false);
+    System.out.println("TEST: now crash");
     crash(writer);
     writer = initIndex(random(), dir, false);
     writer.close();
@@ -88,12 +111,25 @@ public class TestCrash extends LuceneTes
     IndexReader reader = DirectoryReader.open(dir);
     assertTrue(reader.numDocs() < 314);
     reader.close();
+
+    // Make a new dir, copying from the crashed dir, and
+    // open IW on it, to confirm IW "recovers" after a
+    // crash:
+    Directory dir2 = newDirectory(dir);
     dir.close();
+
+    new RandomIndexWriter(random(), dir2).close();
+    dir2.close();
   }
 
   public void testCrashAfterReopen() throws IOException {
     IndexWriter writer = initIndex(random(), false);
     MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+    // We create leftover files because merging could be
+    // running when we crash:
+    dir.setAssertNoUnrefencedFilesOnClose(false);
+
     writer.close();
     writer = initIndex(random(), dir, false);
     assertEquals(314, writer.maxDoc());
@@ -111,7 +147,15 @@ public class TestCrash extends LuceneTes
     IndexReader reader = DirectoryReader.open(dir);
     assertTrue(reader.numDocs() >= 157);
     reader.close();
+
+    // Make a new dir, copying from the crashed dir, and
+    // open IW on it, to confirm IW "recovers" after a
+    // crash:
+    Directory dir2 = newDirectory(dir);
     dir.close();
+
+    new RandomIndexWriter(random(), dir2).close();
+    dir2.close();
   }
 
   public void testCrashAfterClose() throws IOException {

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Wed Oct  3 11:01:13 2012
@@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NoSuchDirectoryException;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java Wed Oct  3 11:01:13 2012
@@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Wed Oct  3 11:01:13 2012
@@ -37,6 +37,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.TrackingDirectoryWrapper;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
@@ -112,6 +113,13 @@ public class TestDoc extends LuceneTestC
       PrintWriter out = new PrintWriter(sw, true);
       
       Directory directory = newFSDirectory(indexDir, null);
+
+      if (directory instanceof MockDirectoryWrapper) {
+        // We create unreferenced files (we don't even write
+        // a segments file):
+        ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+      }
+
       IndexWriter writer = new IndexWriter(
           directory,
           newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
@@ -147,6 +155,13 @@ public class TestDoc extends LuceneTestC
       out = new PrintWriter(sw, true);
 
       directory = newFSDirectory(indexDir, null);
+
+      if (directory instanceof MockDirectoryWrapper) {
+        // We create unreferenced files (we don't even write
+        // a segments file):
+        ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+      }
+
       writer = new IndexWriter(
           directory,
           newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Wed Oct  3 11:01:13 2012
@@ -1070,6 +1070,9 @@ public class TestIndexWriterExceptions e
       fail("segmentInfos failed to retry fallback to correct segments_N file");
     }
     reader.close();
+    
+    // should remove the corrumpted segments_N
+    new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
     dir.close();
   }
 

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Wed Oct  3 11:01:13 2012
@@ -284,6 +284,9 @@ public class TestIndexWriterWithThreads 
         failure.clearDoFail();
         writer.close(false);
       }
+      if (VERBOSE) {
+        System.out.println("TEST: success=" + success);
+      }
 
       if (success) {
         IndexReader reader = DirectoryReader.open(dir);
@@ -341,6 +344,12 @@ public class TestIndexWriterWithThreads 
     }
     @Override
     public void eval(MockDirectoryWrapper dir)  throws IOException {
+
+      // Since we throw exc during abort, eg when IW is
+      // attempting to delete files, we will leave
+      // leftovers: 
+      dir.setAssertNoUnrefencedFilesOnClose(false);
+
       if (doFail) {
         StackTraceElement[] trace = new Exception().getStackTrace();
         boolean sawAbortOrFlushDoc = false;
@@ -405,6 +414,8 @@ public class TestIndexWriterWithThreads 
           if ("flush".equals(trace[i].getMethodName()) && "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
             if (onlyOnce)
               doFail = false;
+            //System.out.println(Thread.currentThread().getName() + ": NOW FAIL: onlyOnce=" + onlyOnce);
+            //new Throwable().printStackTrace(System.out);
             throw new IOException("now failing on purpose");
           }
         }

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java Wed Oct  3 11:01:13 2012
@@ -21,10 +21,12 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.Arrays;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Test;
 

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java Wed Oct  3 11:01:13 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Wed Oct  3 11:01:13 2012
@@ -39,7 +39,6 @@ public class TestRollingUpdates extends 
   public void testRollingUpdates() throws Exception {
     Random random = new Random(random().nextLong());
     final BaseDirectoryWrapper dir = newDirectory();
-    dir.setCheckIndexOnClose(false); // we use a custom codec provider
     final LineFileDocs docs = new LineFileDocs(random, true);
 
     //provider.register(new MemoryCodec());
@@ -130,10 +129,11 @@ public class TestRollingUpdates extends 
     assertEquals(SIZE, w.numDocs());
 
     w.close();
+
+    TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after rolling updates");
+
     docs.close();
     
-    _TestUtil.checkIndex(dir);
-
     // LUCENE-4455:
     SegmentInfos infos = new SegmentInfos();
     infos.read(dir);

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java Wed Oct  3 11:01:13 2012
@@ -17,17 +17,18 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Random;
-import java.io.IOException;
 
+import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.ThreadInterruptedException;
 import org.junit.Test;
@@ -313,6 +314,7 @@ public class TestSnapshotDeletionPolicy 
   public void testRollbackToOldSnapshot() throws Exception {
     int numSnapshots = 2;
     Directory dir = newDirectory();
+
     SnapshotDeletionPolicy sdp = getDeletionPolicy();
     IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
     prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
@@ -325,10 +327,11 @@ public class TestSnapshotDeletionPolicy 
     writer.deleteUnusedFiles();
     assertSnapshotExists(dir, sdp, numSnapshots - 1);
     writer.close();
-    
+
     // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
     String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
     assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
+
     dir.close();
   }
 
@@ -385,6 +388,7 @@ public class TestSnapshotDeletionPolicy 
   @Test
   public void testSnapshotLastCommitTwice() throws Exception {
     Directory dir = newDirectory();
+
     SnapshotDeletionPolicy sdp = getDeletionPolicy();
     IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
     writer.addDocument(new Document());

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java Wed Oct  3 11:01:13 2012
@@ -21,17 +21,18 @@ package org.apache.lucene.index;
 import java.io.IOException;
 import java.util.BitSet;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.HashMap;
 
-import org.apache.lucene.document.Field;
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LuceneTestCase;
 
 /**
  * Test class to illustrate using IndexDeletionPolicy to provide multi-level rollback capability.
@@ -55,13 +56,16 @@ public class TestTransactionRollback ext
     for (Iterator<IndexCommit> iterator = commits.iterator(); iterator.hasNext();) {
       IndexCommit commit =  iterator.next();
       Map<String,String> ud=commit.getUserData();
-      if (ud.size() > 0)
-        if (ud.get("index").endsWith(ids))
-          last=commit;
+      if (ud.size() > 0) {
+        if (ud.get("index").endsWith(ids)) {
+          last = commit;
+        }
+      }
     }
 
-    if (last==null)
+    if (last==null) {
       throw new RuntimeException("Couldn't find commit point "+id);
+    }
 
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(
@@ -74,13 +78,13 @@ public class TestTransactionRollback ext
 
   public void testRepeatedRollBacks() throws Exception {
 
-    int expectedLastRecordId=100;
+    int expectedLastRecordId = 100;
     while (expectedLastRecordId>10) {
-      expectedLastRecordId -=10;
+      expectedLastRecordId -= 10;
       rollBackLast(expectedLastRecordId);
       
       BitSet expecteds = new BitSet(100);
-      expecteds.set(1,(expectedLastRecordId+1),true);
+      expecteds.set(1, (expectedLastRecordId+1), true);
       checkExpecteds(expecteds);
     }
   }
@@ -125,6 +129,7 @@ public class TestTransactionRollback ext
   public void setUp() throws Exception {
     super.setUp();
     dir = newDirectory();
+
     //Build index, of records 1 to 100, committing after each batch of 10
     IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
     IndexWriter w=new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(sdp));
@@ -199,6 +204,7 @@ public class TestTransactionRollback ext
   }
 
   public void testRollbackDeletionPolicy() throws Exception {
+
     for(int i=0;i<2;i++) {
       // Unless you specify a prior commit point, rollback
       // should not work:

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java Wed Oct  3 11:01:13 2012
@@ -215,6 +215,11 @@ public class TestTransactions extends Lu
     dir1.setFailOnOpenInput(false);
     dir2.setFailOnOpenInput(false);
 
+    // We throw exceptions in deleteFile, which creates
+    // leftover files:
+    dir1.setAssertNoUnrefencedFilesOnClose(false);
+    dir2.setAssertNoUnrefencedFilesOnClose(false);
+
     initIndex(dir1);
     initIndex(dir2);
 

Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java?rev=1393408&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java Wed Oct  3 11:01:13 2012
@@ -0,0 +1,58 @@
+package org.apache.lucene.util.junitcompat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collections;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.JUnitCore;
+import org.junit.runner.Result;
+
+// LUCENE-4456: Test that we fail if there are unreferenced files
+public class TestFailIfUnreferencedFiles extends WithNestedTests {
+  public TestFailIfUnreferencedFiles() {
+    super(true);
+  }
+  
+  public static class Nested1 extends WithNestedTests.AbstractNestedTest {
+    public void testDummy() throws Exception {
+      Directory dir = newMockDirectory();
+      IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+      iw.addDocument(new Document());
+      iw.close();
+      IndexOutput output = dir.createOutput("_hello.world", IOContext.DEFAULT);
+      output.writeString("i am unreferenced!");
+      output.close();
+      dir.sync(Collections.singleton("_hello.world"));
+      dir.close();
+    }
+  }
+
+  @Test
+  public void testFailIfUnreferencedFiles() {
+    Result r = JUnitCore.runClasses(Nested1.class);
+    Assert.assertEquals(1, r.getFailureCount());
+  }
+}

Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Wed Oct  3 11:01:13 2012
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -34,6 +35,11 @@ public class TestIndexSplitter extends L
     _TestUtil.rmDir(destDir);
     destDir.mkdirs();
     Directory fsDir = newFSDirectory(dir);
+    // IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
+    // so the unreferenced files are expected.
+    if (fsDir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
+    }
 
     LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
     mergePolicy.setNoCFSRatio(1.0);

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Oct  3 11:01:13 2012
@@ -402,10 +402,16 @@ public class RandomIndexWriter implement
       final int segCount = w.getSegmentCount();
       if (r.nextBoolean() || segCount == 0) {
         // full forceMerge
+        if (LuceneTestCase.VERBOSE) {
+          System.out.println("RIW: doRandomForceMerge(1)");
+        }
         w.forceMerge(1);
       } else {
         // partial forceMerge
         final int limit = _TestUtil.nextInt(r, 1, segCount);
+        if (LuceneTestCase.VERBOSE) {
+          System.out.println("RIW: doRandomForceMerge(" + limit + ")");
+        }
         w.forceMerge(limit);
         assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
       }

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Wed Oct  3 11:01:13 2012
@@ -28,14 +28,18 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoDeletionPolicy;
+import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -204,15 +208,17 @@ public class MockDirectoryWrapper extend
     // first force-close all files, so we can corrupt on windows etc.
     // clone the file map, as these guys want to remove themselves on close.
     Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
-    for (Closeable f : m.keySet())
+    for (Closeable f : m.keySet()) {
       try {
         f.close();
       } catch (Exception ignored) {}
+    }
     
     while(it.hasNext()) {
       String name = it.next();
       int damage = randomState.nextInt(5);
       String action = null;
+
       if (damage == 0) {
         action = "deleted";
         deleteFile(name, true);
@@ -544,6 +550,9 @@ public class MockDirectoryWrapper extend
 
   @Override
   public synchronized void close() throws IOException {
+    // files that we tried to delete, but couldn't because readers were open.
+    // all that matters is that we tried! (they will eventually go away)
+    Set<String> pendingDeletions = new HashSet<String>(openFilesDeleted);
     maybeYield();
     if (openFiles == null) {
       openFiles = new HashMap<String,Integer>();
@@ -574,17 +583,91 @@ public class MockDirectoryWrapper extend
         } 
         _TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
 
+        // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
         if (assertNoUnreferencedFilesOnClose) {
-          // now look for unreferenced files:
-          String[] startFiles = listAll();
-          new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null)).rollback();
-          String[] endFiles = listAll();
+          // now look for unreferenced files: discount ones that we tried to delete but could not
+          Set<String> allFiles = new HashSet<String>(Arrays.asList(listAll()));
+          allFiles.removeAll(pendingDeletions);
+          String[] startFiles = allFiles.toArray(new String[0]);
+          IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
+          iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
+          new IndexWriter(delegate, iwc).rollback();
+          String[] endFiles = delegate.listAll();
+
+          Set<String> startSet = new TreeSet<String>(Arrays.asList(startFiles));
+          Set<String> endSet = new TreeSet<String>(Arrays.asList(endFiles));
+          
+          if (pendingDeletions.contains("segments.gen") && endSet.contains("segments.gen")) {
+            // this is possible if we hit an exception while writing segments.gen, we try to delete it
+            // and it ends out in pendingDeletions (but IFD wont remove this).
+            startSet.add("segments.gen");
+            if (LuceneTestCase.VERBOSE) {
+              System.out.println("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
+            }
+          }
+          
+          // its possible we cannot delete the segments_N on windows if someone has it open and
+          // maybe other files too, depending on timing. normally someone on windows wouldnt have
+          // an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
+          for (String file : pendingDeletions) {
+            if (file.startsWith("segments") && !file.equals("segments.gen") && endSet.contains(file)) {
+              startSet.add(file);
+              if (LuceneTestCase.VERBOSE) {
+                System.out.println("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
+              }
+              try {
+                SegmentInfos sis = new SegmentInfos();
+                sis.read(delegate, file);
+                Set<String> ghosts = new HashSet<String>(sis.files(delegate, false));
+                for (String s : ghosts) {
+                  if (endSet.contains(s) && !startSet.contains(s)) {
+                    assert pendingDeletions.contains(s);
+                    if (LuceneTestCase.VERBOSE) {
+                      System.out.println("MDW: Unreferenced check: Ignoring referenced file: " + s + " " +
+                                         "from " + file + " that we could not delete.");
+                    }
+                    startSet.add(s);
+                  }
+                }
+              } catch (Throwable ignore) {}
+            }
+          }
 
-          Arrays.sort(startFiles);
-          Arrays.sort(endFiles);
+          startFiles = startSet.toArray(new String[0]);
+          endFiles = endSet.toArray(new String[0]);
 
           if (!Arrays.equals(startFiles, endFiles)) {
-            assert false : "unreferenced files: before delete:\n    " + Arrays.toString(startFiles) + "\n  after delete:\n    " + Arrays.toString(endFiles);
+            StringBuilder sb = new StringBuilder();
+            List<String> removed = new ArrayList<String>();
+            for(String fileName : startFiles) {
+              if (!endSet.contains(fileName)) {
+                removed.add(fileName);
+              }
+            }
+
+            List<String> added = new ArrayList<String>();
+            for(String fileName : endFiles) {
+              if (!startSet.contains(fileName)) {
+                added.add(fileName);
+              }
+            }
+
+            String extras;
+            if (removed.size() != 0) {
+              extras = "\n\nThese files were removed: " + removed;
+            } else {
+              extras = "";
+            }
+
+            if (added.size() != 0) {
+              extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
+            }
+
+            if (pendingDeletions.size() != 0) {
+              extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
+            }
+             
+            assert false : "unreferenced files: before delete:\n    " + Arrays.toString(startFiles) + "\n  after delete:\n    " + Arrays.toString(endFiles) + extras;
           }
 
           DirectoryReader ir1 = DirectoryReader.open(this);
@@ -607,7 +690,6 @@ public class MockDirectoryWrapper extend
     if (v != null) {
       if (v.intValue() == 1) {
         openFiles.remove(name);
-        openFilesDeleted.remove(name);
       } else {
         v = Integer.valueOf(v.intValue()-1);
         openFiles.put(name, v);

Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1393408&r1=1393407&r2=1393408&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Oct  3 11:01:13 2012
@@ -948,6 +948,7 @@ public abstract class LuceneTestCase ext
     if (rarely(random)) {
       directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
     }
+
     if (bare) {
       BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
       closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));



Mime
View raw message