lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r1621392 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/ lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene40/ lucene/core/src/java/org/apache/l...
Date Fri, 29 Aug 2014 21:51:42 GMT
Author: mikemccand
Date: Fri Aug 29 21:51:40 2014
New Revision: 1621392

URL: http://svn.apache.org/r1621392
Log:
LUCENE-5904: fix corruption case caused by virus checker after an unclean IW shutdown

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/   (props changed)
    lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/store/TestDirectory.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/store/TestRateLimitedDirectoryWrapper.java
    lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
    lucene/dev/branches/branch_4x/lucene/misc/   (props changed)
    lucene/dev/branches/branch_4x/lucene/misc/src/test/org/apache/lucene/util/fst/TestFSTsMisc.java
    lucene/dev/branches/branch_4x/lucene/replicator/   (props changed)
    lucene/dev/branches/branch_4x/lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java
    lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java
    lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java
    lucene/dev/branches/branch_4x/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
    lucene/dev/branches/branch_4x/lucene/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
    lucene/dev/branches/branch_4x/solr/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/   (props changed)
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockDirectoryFactory.java
    lucene/dev/branches/branch_4x/solr/test-framework/src/java/org/apache/solr/core/MockFSDirectoryFactory.java

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Aug 29 21:51:40 2014
@@ -35,6 +35,16 @@ Bug Fixes
   folder via security manager, and make test temp dirs absolute.
   (Ryan Ernst, Dawid Weiss)
 
+* LUCENE-5904: Fixed a corruption case that can happen when 1)
+  IndexWriter is uncleanly shut-down (OS crash, power loss, etc.), 2)
+  on startup, when a new IndexWriter is created, a virus checker is
+  holding some of the previously written but unused files open and
+  preventing deletion, 3) IndexWriter writes these files again during
+  the course of indexing, then the files can later be deleted, causing
+  corruption.  This case was detected by adding evilness to
+  MockDirectoryWrapper to have it simulate a virus checker holding a
+  file open and preventing deletion (Robert Muir, Mike McCandless)
+
 ======================= Lucene 4.10.0 ======================
 
 New Features

Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Fri Aug 29 21:51:40 2014
@@ -115,10 +115,7 @@ public class SimpleTextSegmentInfoWriter
     } finally {
       if (!success) {
         IOUtils.closeWhileHandlingException(output);
-        try {
-          dir.deleteFile(segFileName);
-        } catch (Throwable t) {
-        }
+        IOUtils.deleteFilesIgnoringExceptions(dir, segFileName);
       } else {
         output.close();
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java Fri Aug 29 21:51:40 2014
@@ -67,7 +67,8 @@ public class Lucene40SegmentInfoWriter e
     } finally {
       if (!success) {
         IOUtils.closeWhileHandlingException(output);
-        si.dir.deleteFile(fileName);
+        // TODO: why must we do this? do we not get tracking dir wrapper?
+        IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
       } else {
         output.close();
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java Fri Aug 29 21:51:40 2014
@@ -65,7 +65,8 @@ public class Lucene46SegmentInfoWriter e
     } finally {
       if (!success) {
         IOUtils.closeWhileHandlingException(output);
-        si.dir.deleteFile(fileName);
+        // TODO: are we doing this outside of the tracking wrapper? why must SIWriter cleanup like this?
+        IOUtils.deleteFilesIgnoringExceptions(si.dir, fileName);
       } else {
         output.close();
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Aug 29 21:51:40 2014
@@ -150,7 +150,7 @@ final class IndexFileDeleter implements 
       // it means the directory is empty, so ignore it.
       files = new String[0];
     }
-    
+
     if (currentSegmentsFile != null) {
       Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
       for (String fileName : files) {
@@ -236,6 +236,9 @@ final class IndexFileDeleter implements 
     // We keep commits list in sorted order (oldest to newest):
     CollectionUtil.timSort(commits);
 
+    // refCounts only includes "normal" filenames (does not include segments.gen, write.lock)
+    inflateGens(segmentInfos, refCounts.keySet(), infoStream);
+
     // Now delete anything with ref count at 0.  These are
     // presumably abandoned files eg due to crash of
     // IndexWriter.
@@ -263,6 +266,83 @@ final class IndexFileDeleter implements 
     deleteCommits();
   }
 
+  /** Set all gens beyond what we currently see in the directory, to avoid double-write in cases where the previous IndexWriter did not
+   *  gracefully close/rollback (e.g. os/machine crashed or lost power). */
+  static void inflateGens(SegmentInfos infos, Collection<String> files, InfoStream infoStream) {
+
+    long maxSegmentGen = Long.MIN_VALUE;
+    int maxSegmentName = Integer.MIN_VALUE;
+
+    // Confusingly, this is the union of liveDocs, field infos, doc values
+    // (and maybe others, in the future) gens.  This is somewhat messy,
+    // since it means DV updates will suddenly write to the next gen after
+    // live docs' gen, for example, but we don't have the APIs to ask the
+    // codec which file is which:
+    Map<String,Long> maxPerSegmentGen = new HashMap<>();
+
+    for(String fileName : files) {
+      if (fileName.equals(IndexFileNames.SEGMENTS_GEN) || fileName.equals(IndexWriter.WRITE_LOCK_NAME)) {
+        // do nothing
+      } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+        try {
+          maxSegmentGen = Math.max(SegmentInfos.generationFromSegmentsFileName(fileName), maxSegmentGen);
+        } catch (NumberFormatException ignore) {
+          // trash file: we have to handle this since we allow anything starting with 'segments' here
+        }
+      } else {
+        String segmentName = IndexFileNames.parseSegmentName(fileName);
+        assert segmentName.startsWith("_"): "wtf? file=" + fileName;
+
+        maxSegmentName = Math.max(maxSegmentName, Integer.parseInt(segmentName.substring(1), Character.MAX_RADIX));
+
+        Long curGen = maxPerSegmentGen.get(segmentName);
+        if (curGen == null) {
+          curGen = 0L;
+        }
+
+        try {
+          curGen = Math.max(curGen, IndexFileNames.parseGeneration(fileName));
+        } catch (NumberFormatException ignore) {
+          // trash file: we have to handle this since codec regex is only so good
+        }
+        maxPerSegmentGen.put(segmentName, curGen);
+      }
+    }
+
+    // Generation is advanced before write:
+    infos.setGeneration(Math.max(infos.getGeneration(), maxSegmentGen));
+    if (infos.counter < 1+maxSegmentName) {
+      if (infoStream.isEnabled("IFD")) {
+        infoStream.message("IFD", "init: inflate infos.counter to " + (1+maxSegmentName) + " vs current=" + infos.counter);
+      }
+      infos.counter = 1+maxSegmentName;
+    }
+
+    for(SegmentCommitInfo info : infos) {
+      Long gen = maxPerSegmentGen.get(info.info.name);
+      assert gen != null;
+      long genLong = gen;
+      if (info.getNextWriteDelGen() < genLong+1) {
+        if (infoStream.isEnabled("IFD")) {
+          infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteDelGen=" + (genLong+1) + " vs current=" + info.getNextWriteDelGen());
+        }
+        info.setNextWriteDelGen(genLong+1);
+      }
+      if (info.getNextWriteFieldInfosGen() < genLong+1) {
+        if (infoStream.isEnabled("IFD")) {
+          infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteFieldInfosGen=" + (genLong+1) + " vs current=" + info.getNextWriteFieldInfosGen());
+        }
+        info.setNextWriteFieldInfosGen(genLong+1);
+      }
+      if (info.getNextWriteDocValuesGen() < genLong+1) {
+        if (infoStream.isEnabled("IFD")) {
+          infoStream.message("IFD", "init: seg=" + info.info.name + " set nextWriteDocValuesGen=" + (genLong+1) + " vs current=" + info.getNextWriteDocValuesGen());
+        }
+        info.setNextWriteDocValuesGen(genLong+1);
+      }
+    }
+  }
+
   private void ensureOpen() throws AlreadyClosedException {
     if (writer == null) {
       throw new AlreadyClosedException("this IndexWriter is closed");
@@ -411,10 +491,17 @@ final class IndexFileDeleter implements 
       deletable = null;
       int size = oldDeletable.size();
       for(int i=0;i<size;i++) {
+        String fileName = oldDeletable.get(i);
         if (infoStream.isEnabled("IFD")) {
-          infoStream.message("IFD", "delete pending file " + oldDeletable.get(i));
+          infoStream.message("IFD", "delete pending file " + fileName);
+        }
+        RefCount rc = refCounts.get(fileName);
+        if (rc != null && rc.count > 0) {
+          // LUCENE-5904: should never happen!  This means we are about to pending-delete a referenced index file
+          assert false: "fileName=" + fileName + " is in pending delete list but also has refCount=" + rc.count;
+        } else {
+          deleteFile(fileName);
         }
-        deleteFile(oldDeletable.get(i));
       }
     }
   }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java Fri Aug 29 21:51:40 2014
@@ -172,6 +172,23 @@ public final class IndexFileNames {
     }
     return filename;
   }
+
+  /** Returns the generation from this file name, or 0 if there is no
+   *  generation. */
+  public static long parseGeneration(String filename) {
+    assert filename.startsWith("_");
+    String parts[] = stripExtension(filename).substring(1).split("_");
+    // 4 cases: 
+    // segment.ext
+    // segment_gen.ext
+    // segment_codec_suffix.ext
+    // segment_gen_codec_suffix.ext
+    if (parts.length == 2 || parts.length == 4) {
+      return Long.parseLong(parts[1], Character.MAX_RADIX);
+    } else {
+      return 0;
+    }
+  }
   
   /**
    * Parses the segment name out of the given file name.

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Fri Aug 29 21:51:40 2014
@@ -192,11 +192,7 @@ public class PersistentSnapshotDeletionP
     } finally {
       if (!success) {
         IOUtils.closeWhileHandlingException(out);
-        try {
-          dir.deleteFile(fileName);
-        } catch (Exception e) {
-          // Suppress so we keep throwing original exception
-        }
+        IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
       } else {
         IOUtils.close(out);
       }
@@ -206,11 +202,8 @@ public class PersistentSnapshotDeletionP
     
     if (nextWriteGen > 0) {
       String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1);
-      try {
-        dir.deleteFile(lastSaveFile);
-      } catch (IOException ioe) {
-        // OK: likely it didn't exist
-      }
+      // exception OK: likely it didn't exist
+      IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile);
     }
 
     nextWriteGen++;
@@ -287,7 +280,7 @@ public class PersistentSnapshotDeletionP
         String curFileName = SNAPSHOTS_PREFIX + genLoaded;
         for(String file : snapshotFiles) {
           if (!curFileName.equals(file)) {
-            dir.deleteFile(file);
+            IOUtils.deleteFilesIgnoringExceptions(dir, file);
           }
         }
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Fri Aug 29 21:51:40 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.store.IOContext
 import org.apache.lucene.store.TrackingDirectoryWrapper;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.MutableBits;
 
 // Used by IndexWriter to hold open SegmentReaders (for
@@ -331,11 +332,7 @@ class ReadersAndUpdates {
         
         // Delete any partially created file(s):
         for (String fileName : trackingDir.getCreatedFiles()) {
-          try {
-            dir.deleteFile(fileName);
-          } catch (Throwable t) {
-            // Ignore so we throw only the first exc
-          }
+          IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
         }
       }
     }
@@ -589,11 +586,7 @@ class ReadersAndUpdates {
         
         // Delete any partially created file(s):
         for (String fileName : trackingDir.getCreatedFiles()) {
-          try {
-            dir.deleteFile(fileName);
-          } catch (Throwable t) {
-            // Ignore so we throw only the first exc
-          }
+          IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
         }
       }
     }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java Fri Aug 29 21:51:40 2014
@@ -148,6 +148,16 @@ public class SegmentCommitInfo {
     nextWriteDelGen++;
   }
   
+  /** Gets the nextWriteDelGen. */
+  long getNextWriteDelGen() {
+    return nextWriteDelGen;
+  }
+  
+  /** Sets the nextWriteDelGen. */
+  void setNextWriteDelGen(long v) {
+    nextWriteDelGen = v;
+  }
+  
   /** Called when we succeed in writing a new FieldInfos generation. */
   void advanceFieldInfosGen() {
     fieldInfosGen = nextWriteFieldInfosGen;
@@ -163,6 +173,16 @@ public class SegmentCommitInfo {
     nextWriteFieldInfosGen++;
   }
   
+  /** Gets the nextWriteFieldInfosGen. */
+  long getNextWriteFieldInfosGen() {
+    return nextWriteFieldInfosGen;
+  }
+  
+  /** Sets the nextWriteFieldInfosGen. */
+  void setNextWriteFieldInfosGen(long v) {
+    nextWriteFieldInfosGen = v;
+  }
+
   /** Called when we succeed in writing a new DocValues generation. */
   void advanceDocValuesGen() {
     docValuesGen = nextWriteDocValuesGen;
@@ -178,6 +198,16 @@ public class SegmentCommitInfo {
     nextWriteDocValuesGen++;
   }
 
+  /** Gets the nextWriteDocValuesGen. */
+  long getNextWriteDocValuesGen() {
+    return nextWriteDocValuesGen;
+  }
+  
+  /** Sets the nextWriteDocValuesGen. */
+  void setNextWriteDocValuesGen(long v) {
+    nextWriteDocValuesGen = v;
+  }
+  
   /** Returns total size in bytes of all files for this
    *  segment. 
    * <p><b>NOTE:</b> This value is not correct for 3.0 segments

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Fri Aug 29 21:51:40 2014
@@ -155,6 +155,7 @@ public final class SegmentInfos implemen
   public static final int FORMAT_SEGMENTS_GEN_CURRENT = FORMAT_SEGMENTS_GEN_CHECKSUM;
 
   /** Used to name new segments. */
+  // TODO: should this be a long ...?
   public int counter;
   
   /** Counts how often the index has been changed.  */
@@ -304,12 +305,7 @@ public final class SegmentInfos implemen
     } catch (Throwable t) {
       // It's OK if we fail to write this file since it's
       // used only as one of the retry fallbacks.
-      try {
-        dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
-      } catch (Throwable t2) {
-        // Ignore; this file is only used in a retry
-        // fallback on init.
-      }
+      IOUtils.deleteFilesIgnoringExceptions(dir, IndexFileNames.SEGMENTS_GEN);
     }
   }
 
@@ -574,13 +570,9 @@ public final class SegmentInfos implemen
           }
         }
 
-        try {
-          // Try not to leave a truncated segments_N file in
-          // the index:
-          directory.deleteFile(segmentsFileName);
-        } catch (Throwable t) {
-          // Suppress so we keep throwing the original exception
-        }
+        // Try not to leave a truncated segments_N file in
+        // the index:
+        IOUtils.deleteFilesIgnoringExceptions(directory, segmentsFileName);
       }
     }
   }
@@ -761,7 +753,7 @@ public final class SegmentInfos implemen
    * commit finishing.
    */
   public abstract static class FindSegmentsFile {
-    
+
     final Directory directory;
 
     /** Sole constructor. */ 
@@ -933,6 +925,9 @@ public final class SegmentInfos implemen
           return v;
         } catch (IOException err) {
 
+          // TODO: we should use the new IO apis in Java7 to get better exceptions on why the open failed.  E.g. we don't want to fall back
+          // if the open failed for a "different" reason (too many open files, access denied) than "the commit was in progress"
+
           // Save the original root cause:
           if (exc == null) {
             exc = err;
@@ -998,6 +993,11 @@ public final class SegmentInfos implemen
     generation = other.generation;
   }
 
+  void setGeneration(long generation) {
+    this.generation = generation;
+    this.lastGeneration = generation;
+  }
+
   final void rollbackCommit(Directory dir) {
     if (pendingSegnOutput != null) {
       // Suppress so we keep throwing the original exception
@@ -1103,11 +1103,7 @@ public final class SegmentInfos implemen
       success = true;
     } finally {
       if (!success) {
-        try {
-          dir.deleteFile(fileName);
-        } catch (Throwable t) {
-          // Suppress so we keep throwing the original exception
-        }
+        IOUtils.deleteFilesIgnoringExceptions(dir, fileName);
       }
     }
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Aug 29 21:51:40 2014
@@ -65,7 +65,9 @@ public class SegmentWriteState {
    *  each of the postings formats it wraps.  If you create
    *  a new {@link PostingsFormat} then any files you
    *  write/read must be derived using this suffix (use
-   *  {@link IndexFileNames#segmentFileName(String,String,String)}). */
+   *  {@link IndexFileNames#segmentFileName(String,String,String)}).
+   *  
+   *  Note: the suffix must be either empty, or be a textual suffix contain exactly two parts (separated by underscore), or be a base36 generation. */
   public final String segmentSuffix;
 
   /** Expert: The fraction of terms in the "dictionary" which should be stored
@@ -99,6 +101,7 @@ public class SegmentWriteState {
     this.segmentInfo = segmentInfo;
     this.fieldInfos = fieldInfos;
     this.termIndexInterval = termIndexInterval;
+    assert assertSegmentSuffix(segmentSuffix);
     this.segmentSuffix = segmentSuffix;
     this.context = context;
   }
@@ -116,4 +119,23 @@ public class SegmentWriteState {
     delCountOnFlush = state.delCountOnFlush;
     liveDocs = state.liveDocs;
   }
+  
+  // currently only used by assert? clean up and make real check?
+  // either its a segment suffix (_X_Y) or its a parseable generation
+  // TODO: this is very confusing how ReadersAndUpdates passes generations via
+  // this mechanism, maybe add 'generation' explicitly to ctor create the 'actual suffix' here?
+  private boolean assertSegmentSuffix(String segmentSuffix) {
+    assert segmentSuffix != null;
+    if (!segmentSuffix.isEmpty()) {
+      int numParts = segmentSuffix.split("_").length;
+      if (numParts == 2) {
+        return true;
+      } else if (numParts == 1) {
+        Long.parseLong(segmentSuffix, Character.MAX_RADIX);
+        return true;
+      }
+      return false; // invalid
+    }
+    return true;
+  }
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Fri Aug 29 21:51:40 2014
@@ -203,7 +203,8 @@ final class CompoundFileWriter implement
       if (success) {
         IOUtils.close(is);
         // copy successful - delete file
-        fileEntry.dir.deleteFile(fileEntry.file);
+        // if we can't we rely on IFD to pick up and retry
+        IOUtils.deleteFilesIgnoringExceptions(fileEntry.dir, fileEntry.file);
       } else {
         IOUtils.closeWhileHandlingException(is);
       }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestCompressingStoredFieldsFormat.java Fri Aug 29 21:51:40 2014
@@ -30,6 +30,7 @@ import org.apache.lucene.index.BaseStore
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.junit.Test;
 
 import com.carrotsearch.randomizedtesting.annotations.Repeat;
@@ -46,6 +47,10 @@ public class TestCompressingStoredFields
   @Test(expected=IllegalArgumentException.class)
   public void testDeletePartiallyWrittenFilesIfAbort() throws IOException {
     Directory dir = newDirectory();
+    // test explicitly needs files to always be actually deleted
+    if (dir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
     iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
     iwConf.setCodec(CompressingCodec.randomInstance(random()));

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Aug 29 21:51:40 2014
@@ -1094,7 +1094,8 @@ public class TestAddIndexes extends Luce
     
     IndexReader[] readers = new IndexReader[] { DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1]) };
     
-    Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+    MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+    dir.setEnableVirusScanner(false); // we check for specific list of files
     IndexWriterConfig conf = new IndexWriterConfig(Version.LATEST, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(true));
     MergePolicy lmp = conf.getMergePolicy();
     // Force creation of CFS:

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Fri Aug 29 21:51:40 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -38,6 +39,10 @@ import org.apache.lucene.util.TestUtil;
 public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
   public void test() throws Exception {
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
+      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+    }
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     conf.setCodec(new Lucene410Codec());
     RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Fri Aug 29 21:51:40 2014
@@ -28,6 +28,7 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -38,6 +39,12 @@ import org.apache.lucene.util.TestUtil;
 public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
   public void test() throws Exception {
     Directory dir = newDirectory();
+
+    if (dir instanceof MockDirectoryWrapper) {
+      // Else we might remove .cfe but not the corresponding .cfs, causing false exc when trying to verify headers:
+      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
+    }
+
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     conf.setCodec(new Lucene410Codec());
     RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Fri Aug 29 21:51:40 2014
@@ -54,6 +54,7 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.store.SimpleFSDirectory;
@@ -994,6 +995,11 @@ public class TestBackwardsCompatibility 
         System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" +name);
       }
       Directory dir = newDirectory(oldIndexDirs.get(name));
+      if (dir instanceof MockDirectoryWrapper) {
+        // we need to ensure we delete old commits for this test,
+        // otherwise IndexUpgrader gets angry
+        ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+      }
 
       assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
 
@@ -1025,6 +1031,8 @@ public class TestBackwardsCompatibility 
       // determine count of segments in modified index
       final int origSegCount = getNumberOfSegments(dir);
       
+      // ensure there is only one commit
+      assertEquals(1, DirectoryReader.listCommits(dir).size());
       newIndexUpgrader(dir).upgrade();
 
       final int segCount = checkAllSegmentsUpgraded(dir);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Fri Aug 29 21:51:40 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Sorted
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -1163,6 +1164,10 @@ public class TestBinaryDocValuesUpdates 
 
   public void testDeleteUnusedUpdatesFiles() throws Exception {
     Directory dir = newDirectory();
+    // test explicitly needs files to always be actually deleted
+    if (dir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
     

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java Fri Aug 29 21:51:40 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -28,7 +29,9 @@ import org.apache.lucene.util.TestUtil;
 
 public class TestCodecHoldsOpenFiles extends LuceneTestCase {
   public void test() throws Exception {
-    Directory d = newDirectory();
+    BaseDirectoryWrapper d = newDirectory();
+    d.setCheckIndexOnClose(false);
+    // we nuke files, but verify the reader still works
     RandomIndexWriter w = new RandomIndexWriter(random(), d);
     int numDocs = atLeast(100);
     for(int i=0;i<numDocs;i++) {
@@ -38,11 +41,13 @@ public class TestCodecHoldsOpenFiles ext
     }
 
     IndexReader r = w.getReader();
+    w.commit();
     w.close();
 
     for(String fileName : d.listAll()) {
       try {
         d.deleteFile(fileName);
+        // may succeed, e.g. if the file is completely read into RAM.
       } catch (IOException ioe) {
         // ignore: this means codec (correctly) is holding
         // the file open

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java Fri Aug 29 21:51:40 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -709,6 +710,10 @@ public class TestCompoundFile extends Lu
   
   public void testReadNestedCFP() throws IOException {
     Directory newDir = newDirectory();
+    // manually manipulates directory
+    if (newDir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)newDir).setEnableVirusScanner(false);
+    }
     CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random()), true);
     CompoundFileDirectory nested = new CompoundFileDirectory(newDir, "b.cfs", newIOContext(random()), true);
     IndexOutput out = nested.createOutput("b.xyz", newIOContext(random()));

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Fri Aug 29 21:51:40 2014
@@ -178,6 +178,10 @@ public class TestConcurrentMergeSchedule
 
   public void testNoExtraFiles() throws IOException {
     Directory directory = newDirectory();
+    if (directory instanceof MockDirectoryWrapper) {
+      // test uses IW unref'ed helper which is unaware of retries
+      ((MockDirectoryWrapper)directory).setEnableVirusScanner(false);
+    }
     IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random()))
                                                       .setMaxBufferedDocs(2));
 

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDeletionPolicy.java Fri Aug 29 21:51:40 2014
@@ -34,6 +34,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.Version;
@@ -222,6 +223,10 @@ public class TestDeletionPolicy extends 
     final double SECONDS = 2.0;
 
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // test manually deletes files
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
         .setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
     MergePolicy mp = conf.getMergePolicy();
@@ -315,6 +320,10 @@ public class TestDeletionPolicy extends 
       boolean useCompoundFile = (pass % 2) != 0;
 
       Directory dir = newDirectory();
+      if (dir instanceof MockDirectoryWrapper) {
+        // test manually deletes files
+        ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+      }
 
       IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
           .setIndexDeletionPolicy(new KeepAllDeletionPolicy(dir))
@@ -563,6 +572,10 @@ public class TestDeletionPolicy extends 
       boolean useCompoundFile = (pass % 2) != 0;
 
       Directory dir = newDirectory();
+      if (dir instanceof MockDirectoryWrapper) {
+        // test manually deletes files
+        ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+      }
 
       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
       for(int j=0;j<N+1;j++) {
@@ -624,6 +637,10 @@ public class TestDeletionPolicy extends 
       boolean useCompoundFile = (pass % 2) != 0;
 
       Directory dir = newDirectory();
+      if (dir instanceof MockDirectoryWrapper) {
+        // test manually deletes files
+        ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+      }
       IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
           .setOpenMode(OpenMode.CREATE)
           .setIndexDeletionPolicy(new KeepLastNDeletionPolicy(N))

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java Fri Aug 29 21:51:40 2014
@@ -630,6 +630,10 @@ public class TestDirectoryReaderReopen e
 
   public void testOverDecRefDuringReopen() throws Exception {
     MockDirectoryWrapper dir = newMockDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // ensure we produce enough of our exceptions
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
 
     IndexWriterConfig iwc = new IndexWriterConfig(Version.LATEST, new MockAnalyzer(random()));
     iwc.setCodec(Codec.forName("Lucene410"));

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Fri Aug 29 21:51:40 2014
@@ -119,6 +119,8 @@ public class TestDoc extends LuceneTestC
         // We create unreferenced files (we don't even write
         // a segments file):
         ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+        // this test itself deletes files (has no retry mechanism)
+        ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
       }
 
       IndexWriter writer = new IndexWriter(
@@ -161,6 +163,8 @@ public class TestDoc extends LuceneTestC
         // We create unreferenced files (we don't even write
         // a segments file):
         ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+        // this test itself deletes files (has no retry mechanism)
+        ((MockDirectoryWrapper) directory).setEnableVirusScanner(false);
       }
 
       writer = new IndexWriter(

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Fri Aug 29 21:51:40 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.*;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
@@ -26,10 +27,13 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Version;
 
 /*
   Verify we can read the pre-2.1 file format, do searches
@@ -42,6 +46,8 @@ public class TestIndexFileDeleter extend
     Directory dir = newDirectory();
     if (dir instanceof MockDirectoryWrapper) {
       ((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
+      // ensure we actually delete files
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
     }
 
     MergePolicy mergePolicy = newLogMergePolicy(true, 10);
@@ -203,4 +209,194 @@ public class TestIndexFileDeleter extend
     doc.add(newStringField("id", Integer.toString(id), Field.Store.NO));
     writer.addDocument(doc);
   }
+  
+  public void testVirusScannerDoesntCorruptIndex() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    dir.setPreventDoubleWrite(false); // we arent trying to test this
+    dir.setEnableVirusScanner(false); // we have our own to make test reproduce always
+    
+    // add empty commit
+    new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null)).close();
+    // add a trash unreferenced file
+    dir.createOutput("_0.si", IOContext.DEFAULT).close();
+
+    // start virus scanner
+    final AtomicBoolean stopScanning = new AtomicBoolean();
+    dir.failOn(new MockDirectoryWrapper.Failure() {
+      @Override
+      public void eval(MockDirectoryWrapper dir) throws IOException {
+        if (stopScanning.get()) {
+          return;
+        }
+        for (StackTraceElement f : new Exception().getStackTrace()) {
+          if ("deleteFile".equals(f.getMethodName())) {
+            throw new IOException("temporarily cannot delete file");
+          }
+        }
+      }
+    });
+    
+    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null));
+    iw.addDocument(new Document());
+    // stop virus scanner
+    stopScanning.set(true);
+    iw.commit();
+    iw.close();
+    dir.close();
+  }
+  
+  public void testNoSegmentsDotGenInflation() throws IOException {
+    Directory dir = newMockDirectory();
+    
+    // empty commit
+    new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null)).close();   
+    
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(1, sis.getGeneration());
+    
+    // no inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(1, sis.getGeneration());
+
+    dir.close();
+  }
+  
+  public void testSegmentsInflation() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    dir.setCheckIndexOnClose(false); // TODO: allow falling back more than one commit
+    
+    // empty commit
+    new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null)).close();   
+    
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(1, sis.getGeneration());
+    
+    // add trash commit
+    dir.createOutput(IndexFileNames.SEGMENTS + "_2", IOContext.DEFAULT).close();
+    
+    // ensure inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(2, sis.getGeneration());
+    
+    // add another trash commit
+    dir.createOutput(IndexFileNames.SEGMENTS + "_4", IOContext.DEFAULT).close();
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(4, sis.getGeneration());
+
+    dir.close();
+  }
+  
+  public void testSegmentNameInflation() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    
+    // empty commit
+    new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null)).close();   
+    
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(0, sis.counter);
+    
+    // no inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(0, sis.counter);
+    
+    // add trash per-segment file
+    dir.createOutput(IndexFileNames.segmentFileName("_0", "", "foo"), IOContext.DEFAULT).close();
+    
+    // ensure inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(1, sis.counter);
+    
+    // add trash per-segment file
+    dir.createOutput(IndexFileNames.segmentFileName("_3", "", "foo"), IOContext.DEFAULT).close();
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(4, sis.counter);
+    
+    // ensure we write _4 segment next
+    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null));
+    iw.addDocument(new Document());
+    iw.commit();
+    iw.close();
+    sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals("_4", sis.info(0).info.name);
+    assertEquals(5, sis.counter);
+    
+    dir.close();
+  }
+  
+  public void testGenerationInflation() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    
+    // initial commit
+    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null));
+    iw.addDocument(new Document());
+    iw.commit();
+    iw.close();   
+    
+    // no deletes: start at 1
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(1, sis.info(0).getNextDelGen());
+    
+    // no inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(1, sis.info(0).getNextDelGen());
+    
+    // add trash per-segment deletes file
+    dir.createOutput(IndexFileNames.fileNameFromGeneration("_0", "del", 2), IOContext.DEFAULT).close();
+    
+    // ensure inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(3, sis.info(0).getNextDelGen());
+    
+    dir.close();
+  }
+  
+  public void testTrashyFile() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    dir.setCheckIndexOnClose(false); // TODO: maybe handle such trash better elsewhere...
+    
+    // empty commit
+    new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null)).close();   
+    
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(1, sis.getGeneration());
+    
+    // add trash file
+    dir.createOutput(IndexFileNames.SEGMENTS + "_", IOContext.DEFAULT).close();
+    
+    // no inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(1, sis.getGeneration());
+
+    dir.close();
+  }
+  
+  public void testTrashyGenFile() throws IOException {
+    MockDirectoryWrapper dir = newMockDirectory();
+    
+    // initial commit
+    IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, null));
+    iw.addDocument(new Document());
+    iw.commit();
+    iw.close();   
+    
+    // no deletes: start at 1
+    SegmentInfos sis = new SegmentInfos();
+    sis.read(dir);
+    assertEquals(1, sis.info(0).getNextDelGen());
+    
+    // add trash file
+    dir.createOutput("_1_A", IOContext.DEFAULT).close();
+    
+    // no inflation
+    IndexFileDeleter.inflateGens(sis, Arrays.asList(dir.listAll()), InfoStream.getDefault());
+    assertEquals(1, sis.info(0).getNextDelGen());
+
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Aug 29 21:51:40 2014
@@ -173,7 +173,12 @@ public class TestIndexWriter extends Luc
 
 
 
+    // TODO: we have the logic in MDW to do this check, and its better, because it knows about files it tried
+    // to delete but couldn't: we should replace this!!!!
     public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
+      if (dir instanceof MockDirectoryWrapper) {
+        assertFalse("test is broken: should disable virus scanner", ((MockDirectoryWrapper)dir).getEnableVirusScanner());
+      }
       String[] startFiles = dir.listAll();
       new IndexWriter(dir, new IndexWriterConfig(Version.LATEST, new MockAnalyzer(random()))).rollback();
       String[] endFiles = dir.listAll();
@@ -311,21 +316,31 @@ public class TestIndexWriter extends Luc
               .setRAMBufferSizeMB(0.000001)
               .setMergePolicy(newLogMergePolicy(10))
       );
-      int lastNumFile = dir.listAll().length;
+      int lastNumSegments = getSegmentCount(dir);
       for(int j=0;j<9;j++) {
         Document doc = new Document();
         doc.add(newField("field", "aaa" + j, storedTextType));
         writer.addDocument(doc);
-        int numFile = dir.listAll().length;
         // Verify that with a tiny RAM buffer we see new
         // segment after every doc
-        assertTrue(numFile > lastNumFile);
-        lastNumFile = numFile;
+        int numSegments = getSegmentCount(dir);
+        assertTrue(numSegments > lastNumSegments);
+        lastNumSegments = numSegments;
       }
       writer.close();
       dir.close();
     }
 
+    /** Returns how many unique segment names are in the directory. */
+    private static int getSegmentCount(Directory dir) throws IOException {
+      Set<String> segments = new HashSet<>();
+      for(String file : dir.listAll()) {
+        segments.add(IndexFileNames.parseSegmentName(file));
+      }
+
+      return segments.size();
+    }
+
     // Make sure it's OK to change RAM buffer size and
     // maxBufferedDocs in a write session
     public void testChangingRAMBuffer() throws IOException {
@@ -1011,8 +1026,10 @@ public class TestIndexWriter extends Luc
     final Directory adder;
     final ByteArrayOutputStream bytesLog = new ByteArrayOutputStream();
     final PrintStream log = new PrintStream(bytesLog, true, IOUtils.UTF_8);
-    
-    IndexerThreadInterrupt() throws IOException {
+    final int id;
+
+    IndexerThreadInterrupt(int id) throws IOException {
+      this.id = id;
       this.random = new Random(random().nextLong());
       // make a little directory for addIndexes
       // LUCENE-2239: won't work with NIOFS/MMAP
@@ -1161,7 +1178,7 @@ public class TestIndexWriter extends Luc
           // on!!  This test doesn't repro easily so when
           // Jenkins hits a fail we need to study where the
           // interrupts struck!
-          log.println("TEST: got interrupt");
+          log.println("TEST thread " + id + ": got interrupt");
           re.printStackTrace(log);
           Throwable e = re.getCause();
           assertTrue(e instanceof InterruptedException);
@@ -1169,7 +1186,7 @@ public class TestIndexWriter extends Luc
             break;
           }
         } catch (Throwable t) {
-          log.println("FAILED; unexpected exception");
+          log.println("thread " + id + " FAILED; unexpected exception");
           t.printStackTrace(log);
           failed = true;
           break;
@@ -1177,11 +1194,11 @@ public class TestIndexWriter extends Luc
       }
 
       if (VERBOSE) {
-        log.println("TEST: now finish failed=" + failed);
+        log.println("TEST: thread " + id + ": now finish failed=" + failed);
       }
       if (!failed) {
         if (VERBOSE) {
-          log.println("TEST: now rollback");
+          log.println("TEST: thread " + id + ": now rollback");
         }
         // clear interrupt state:
         Thread.interrupted();
@@ -1197,7 +1214,7 @@ public class TestIndexWriter extends Luc
           TestUtil.checkIndex(dir);
         } catch (Exception e) {
           failed = true;
-          log.println("CheckIndex FAILED: unexpected exception");
+          log.println("thread " + id + ": CheckIndex FAILED: unexpected exception");
           e.printStackTrace(log);
         }
         try {
@@ -1206,25 +1223,27 @@ public class TestIndexWriter extends Luc
           r.close();
         } catch (Exception e) {
           failed = true;
-          log.println("DirectoryReader.open FAILED: unexpected exception");
+          log.println("thread " + id + ": DirectoryReader.open FAILED: unexpected exception");
           e.printStackTrace(log);
         }
       }
       try {
         IOUtils.close(dir);
       } catch (IOException e) {
-        throw new RuntimeException(e);
+        failed = true;
+        throw new RuntimeException("thread " + id, e);
       }
       try {
         IOUtils.close(adder);
       } catch (IOException e) {
-        throw new RuntimeException(e);
+        failed = true;
+        throw new RuntimeException("thread " + id, e);
       }
     }
   }
 
   public void testThreadInterruptDeadlock() throws Exception {
-    IndexerThreadInterrupt t = new IndexerThreadInterrupt();
+    IndexerThreadInterrupt t = new IndexerThreadInterrupt(1);
     t.setDaemon(true);
     t.start();
 
@@ -1258,11 +1277,11 @@ public class TestIndexWriter extends Luc
   
   /** testThreadInterruptDeadlock but with 2 indexer threads */
   public void testTwoThreadsInterruptDeadlock() throws Exception {
-    IndexerThreadInterrupt t1 = new IndexerThreadInterrupt();
+    IndexerThreadInterrupt t1 = new IndexerThreadInterrupt(1);
     t1.setDaemon(true);
     t1.start();
     
-    IndexerThreadInterrupt t2 = new IndexerThreadInterrupt();
+    IndexerThreadInterrupt t2 = new IndexerThreadInterrupt(2);
     t2.setDaemon(true);
     t2.start();
 
@@ -1292,8 +1311,13 @@ public class TestIndexWriter extends Luc
     t2.finish = true;
     t1.join();
     t2.join();
-    assertFalse(t1.failed);
-    assertFalse(t2.failed);
+    if (t1.failed) {
+      System.out.println("Thread1 failed:\n" + new String(t1.bytesLog.toString("UTF-8")));
+    }
+    if (t2.failed) {
+      System.out.println("Thread2 failed:\n" + new String(t2.bytesLog.toString("UTF-8")));
+    }
+    assertFalse(t1.failed || t2.failed);
   }
 
 
@@ -1408,7 +1432,8 @@ public class TestIndexWriter extends Luc
 
   public void testDeleteUnusedFiles() throws Exception {
     for(int iter=0;iter<2;iter++) {
-      Directory dir = newMockDirectory(); // relies on windows semantics
+      MockDirectoryWrapper dir = newMockDirectory(); // relies on windows semantics
+      dir.setEnableVirusScanner(false); // but ensures files are actually deleted
 
       MergePolicy mergePolicy = newLogMergePolicy(true);
       
@@ -1497,10 +1522,14 @@ public class TestIndexWriter extends Luc
     }
   }
 
-  public void testDeleteUnsedFiles2() throws Exception {
+  public void testDeleteUnusedFiles2() throws Exception {
     // Validates that iw.deleteUnusedFiles() also deletes unused index commits
     // in case a deletion policy which holds onto commits is used.
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // otherwise the delete of old commit might not actually succeed temporarily.
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                                 .setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
     SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
@@ -1553,6 +1582,9 @@ public class TestIndexWriter extends Luc
     // indexed, flushed (but not committed) and then IW rolls back, then no
     // files are left in the Directory.
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                                 .setMaxBufferedDocs(2)
                                                 .setMergePolicy(newLogMergePolicy())
@@ -1639,6 +1671,10 @@ public class TestIndexWriter extends Luc
   public void testNoUnwantedTVFiles() throws Exception {
 
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // test uses IW unref'ed check which is unaware of retries
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                                      .setRAMBufferSizeMB(0.01)
                                                      .setMergePolicy(newLogMergePolicy()));
@@ -1824,7 +1860,8 @@ public class TestIndexWriter extends Luc
 
   public void testDeleteAllNRTLeftoverFiles() throws Exception {
 
-    Directory d = new MockDirectoryWrapper(random(), new RAMDirectory());
+    MockDirectoryWrapper d = new MockDirectoryWrapper(random(), new RAMDirectory());
+    d.setEnableVirusScanner(false); // needs for files to actually be deleted
     IndexWriter w = new IndexWriter(d, new IndexWriterConfig(Version.LATEST, new MockAnalyzer(random())));
     Document doc = new Document();
     for(int i = 0; i < 20; i++) {
@@ -1949,6 +1986,10 @@ public class TestIndexWriter extends Luc
   // LUCENE-3872
   public void testPrepareCommitThenRollback() throws Exception {
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // indexExists might return true if virus scanner prevents deletions 
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter w = new IndexWriter(dir,
                                     new IndexWriterConfig(Version.LATEST, new MockAnalyzer(random())));
 
@@ -2698,6 +2739,9 @@ public class TestIndexWriter extends Luc
     assumeFalse("this test can't run on Windows", Constants.WINDOWS);
 
     MockDirectoryWrapper dir = newMockDirectory();
+    
+    // don't act like windows either, or the test won't simulate the condition
+    dir.setEnableVirusScanner(false);
 
     // Allow deletion of still open files:
     dir.setNoDeleteOpenFile(false);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Fri Aug 29 21:51:40 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -94,6 +95,10 @@ public class TestIndexWriterCommit exten
    */
   public void testCommitOnCloseAbort() throws IOException {
     Directory dir = newDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // test uses IW unref'ed check which is unaware of retries
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                                 .setMaxBufferedDocs(10));
     for (int i = 0; i < 14; i++) {
@@ -185,6 +190,11 @@ public class TestIndexWriterCommit exten
     final String contentFormat = TestUtil.getPostingsFormat("content");
     assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory"));
     MockDirectoryWrapper dir = newMockDirectory();
+    if (dir instanceof MockDirectoryWrapper) {
+      // the virus scanner can use up too much disk space :)
+      // an alternative is to expose MDW.triedToDelete and discount it
+      dir.setEnableVirusScanner(false);
+    }
     Analyzer analyzer;
     if (random().nextBoolean()) {
       // no payloads
@@ -270,6 +280,10 @@ public class TestIndexWriterCommit exten
     if (dir instanceof MockDirectoryWrapper) {
       ((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
     }
+    if (dir instanceof MockDirectoryWrapper) {
+      // test uses IW unref'ed check which is unaware of retries
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter writer = new IndexWriter(
         dir,
         newIndexWriterConfig(new MockAnalyzer(random()))
@@ -557,8 +571,13 @@ public class TestIndexWriterCommit exten
   // LUCENE-1274: test writer.prepareCommit()
   public void testPrepareCommitRollback() throws IOException {
     Directory dir = newDirectory();
+
+    MockDirectoryWrapper mockDir;
     if (dir instanceof MockDirectoryWrapper) {
-      ((MockDirectoryWrapper)dir).setPreventDoubleWrite(false);
+      mockDir = (MockDirectoryWrapper) dir;
+      mockDir.setPreventDoubleWrite(false);
+    } else {
+      mockDir = null;
     }
 
     IndexWriter writer = new IndexWriter(
@@ -569,8 +588,9 @@ public class TestIndexWriterCommit exten
     );
     writer.commit();
 
-    for (int i = 0; i < 23; i++)
+    for (int i = 0; i < 23; i++) {
       TestIndexWriter.addDoc(writer);
+    }
 
     DirectoryReader reader = DirectoryReader.open(dir);
     assertEquals(0, reader.numDocs());
@@ -580,7 +600,14 @@ public class TestIndexWriterCommit exten
     IndexReader reader2 = DirectoryReader.open(dir);
     assertEquals(0, reader2.numDocs());
 
+    // We need to let IW delete the partial segments_N that was written in prepareCommit, else we get a false fail below:
+    if (mockDir != null) {
+      mockDir.setEnableVirusScanner(false);
+    }
     writer.rollback();
+    if (mockDir != null) {
+      mockDir.setEnableVirusScanner(true);
+    }
 
     IndexReader reader3 = DirectoryReader.openIfChanged(reader);
     assertNull(reader3);
@@ -589,9 +616,12 @@ public class TestIndexWriterCommit exten
     reader.close();
     reader2.close();
 
+    // System.out.println("TEST: after rollback: " + Arrays.toString(dir.listAll()));
+
     writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
-    for (int i = 0; i < 17; i++)
+    for (int i = 0; i < 17; i++) {
       TestIndexWriter.addDoc(writer);
+    }
 
     reader = DirectoryReader.open(dir);
     assertEquals(0, reader.numDocs());

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Aug 29 21:51:40 2014
@@ -509,6 +509,8 @@ public class TestIndexWriterDelete exten
     MockDirectoryWrapper startDir = newMockDirectory();
     // TODO: find the resource leak that only occurs sometimes here.
     startDir.setNoDeleteOpenFile(false);
+    // test uses IW unref'ed helper which is unaware of retries
+    startDir.setEnableVirusScanner(false);
     IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     for (int i = 0; i < 157; i++) {
       Document d = new Document();
@@ -536,6 +538,8 @@ public class TestIndexWriterDelete exten
       MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory(startDir, newIOContext(random())));
       dir.setPreventDoubleWrite(false);
       dir.setAllowRandomFileNotFoundException(false);
+      // test uses IW unref'ed helper which is unaware of retries
+      dir.setEnableVirusScanner(false);
       IndexWriter modifier = new IndexWriter(dir,
                                              newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false))
                                              .setMaxBufferedDocs(1000)
@@ -752,7 +756,7 @@ public class TestIndexWriterDelete exten
                 System.out.println("TEST: mock failure: now fail");
                 new Throwable().printStackTrace(System.out);
               }
-              throw new IOException("fail after applyDeletes");
+              throw new RuntimeException("fail after applyDeletes");
             }
           }
           if (!failed) {
@@ -856,7 +860,7 @@ public class TestIndexWriterDelete exten
         System.out.println("TEST: now commit for failure");
       }
       modifier.commit();
-    } catch (IOException ioe) {
+    } catch (RuntimeException ioe) {
       // expected
       failed = true;
     }
@@ -907,6 +911,8 @@ public class TestIndexWriterDelete exten
     String[] text = { "Amsterdam", "Venice" };
 
     MockDirectoryWrapper dir = newMockDirectory();
+    // test uses IW unref'ed helper which is unaware of retries
+    dir.setEnableVirusScanner(false);
     IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     modifier.commit();
     dir.failOn(failure.reset());

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Aug 29 21:51:40 2014
@@ -950,6 +950,7 @@ public class TestIndexWriterExceptions e
     for (FailOnlyInCommit failure : failures) {
       MockDirectoryWrapper dir = newMockDirectory();
       dir.setFailOnCreateOutput(false);
+      dir.setEnableVirusScanner(false); // we check for specific list of files
       IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
       Document doc = new Document();
       doc.add(newTextField("field", "a field", Field.Store.YES));
@@ -1171,6 +1172,10 @@ public class TestIndexWriterExceptions e
   public void testSimulatedCorruptIndex1() throws IOException {
       BaseDirectoryWrapper dir = newDirectory();
       dir.setCheckIndexOnClose(false); // we are corrupting it!
+      if (dir instanceof MockDirectoryWrapper) {
+        // we want to ensure our corruption always succeeds!
+        ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+      }
 
       IndexWriter writer = null;
 
@@ -1219,6 +1224,10 @@ public class TestIndexWriterExceptions e
   public void testSimulatedCorruptIndex2() throws IOException {
     BaseDirectoryWrapper dir = newDirectory();
     dir.setCheckIndexOnClose(false); // we are corrupting it!
+    if (dir instanceof MockDirectoryWrapper) {
+      // we want to ensure our corruption always succeeds!
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriter writer = null;
 
     writer  = new IndexWriter(

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Fri Aug 29 21:51:40 2014
@@ -61,6 +61,7 @@ public class TestIndexWriterOnDiskFull e
           System.out.println("TEST: cycle: diskFree=" + diskFree);
         }
         MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
+        dir.setEnableVirusScanner(false); // currently uses the IW unreferenced files method, unaware of retries
         dir.setMaxSizeInBytes(diskFree);
         IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
         MergeScheduler ms = writer.getConfig().getMergeScheduler();
@@ -532,6 +533,7 @@ public class TestIndexWriterOnDiskFull e
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                                                 .setMaxBufferedDocs(2)
                                                 .setMergeScheduler(new ConcurrentMergeScheduler()));
+    writer.commit(); // empty commit, to not create confusing situation with first commit
     dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
     final Document doc = new Document();
     FieldType customType = new FieldType(TextField.TYPE_STORED);

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfMemory.java Fri Aug 29 21:51:40 2014
@@ -100,6 +100,7 @@ public class TestIndexWriterOutOfMemory 
         int numDocs = atLeast(2000);
       
         IndexWriter iw = new IndexWriter(dir, conf);
+        iw.commit(); // ensure there is always a commit
 
         final Random r = new Random(random().nextLong());
         dir.failOn(new Failure() {

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Fri Aug 29 21:51:40 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Sorted
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NRTCachingDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -1160,6 +1161,10 @@ public class TestNumericDocValuesUpdates
   @Test
   public void testDeleteUnusedUpdatesFiles() throws Exception {
     Directory dir = newDirectory();
+    // test explicitly needs files to always be actually deleted
+    if (dir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
     IndexWriter writer = new IndexWriter(dir, conf);
     

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java Fri Aug 29 21:51:40 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.document.TextFi
 import org.apache.lucene.index.FieldInfo.IndexOptions;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -189,6 +190,10 @@ public class TestOmitPositions extends L
   // Verifies no *.prx exists when all fields omit term positions:
   public void testNoPrxFile() throws Throwable {
     Directory ram = newDirectory();
+    if (ram instanceof MockDirectoryWrapper) {
+      // we verify some files get deleted
+      ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
+    }
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
                                                 .setMaxBufferedDocs(3)

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Fri Aug 29 21:51:40 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.search.TermStatistics;
 import org.apache.lucene.search.similarities.TFIDFSimilarity;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -219,6 +220,10 @@ public class TestOmitTf extends LuceneTe
   // Verifies no *.prx exists when all fields omit term freq:
   public void testNoPrxFile() throws Throwable {
     Directory ram = newDirectory();
+    if (ram instanceof MockDirectoryWrapper) {
+      // we verify some files get deleted
+      ((MockDirectoryWrapper)ram).setEnableVirusScanner(false);
+    }
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriter writer = new IndexWriter(ram, newIndexWriterConfig(analyzer)
                                                 .setMaxBufferedDocs(3)

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java Fri Aug 29 21:51:40 2014
@@ -50,6 +50,7 @@ public class TestPersistentSnapshotDelet
   public void testExistingSnapshots() throws Exception {
     int numSnapshots = 3;
     MockDirectoryWrapper dir = newMockDirectory();
+    dir.setEnableVirusScanner(false); // test relies on files actually being deleted
     IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
     PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
     assertNull(psdp.getLastSaveFile());

Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1621392&r1=1621391&r2=1621392&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Fri Aug 29 21:51:40 2014
@@ -39,8 +39,12 @@ public class TestRollingUpdates extends 
   public void testRollingUpdates() throws Exception {
     Random random = new Random(random().nextLong());
     final BaseDirectoryWrapper dir = newDirectory();
+    // test checks for no unref'ed files with the IW helper method, which isn't aware of "tried to delete files"
+    if (dir instanceof MockDirectoryWrapper) {
+      ((MockDirectoryWrapper)dir).setEnableVirusScanner(false);
+    }
     final LineFileDocs docs = new LineFileDocs(random, defaultCodecSupportsDocValues());
-
+    
     //provider.register(new MemoryCodec());
     if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random().nextBoolean()) {
       Codec.setDefault(TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat(random().nextBoolean(), random.nextFloat())));



Mime
View raw message