lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r517599 [2/2] - in /lucene/java/trunk: ./ src/java/org/apache/lucene/index/ src/site/src/documentation/content/xdocs/ src/test/org/apache/lucene/index/ src/test/org/apache/lucene/store/
Date Tue, 13 Mar 2007 09:06:23 GMT
Modified: lucene/java/trunk/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/SegmentInfos.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/SegmentInfos.java Tue Mar 13 02:06:22 2007
@@ -88,16 +88,9 @@
     for (int i = 0; i < files.length; i++) {
       String file = files[i];
       if (file.startsWith(IndexFileNames.SEGMENTS) && !file.equals(IndexFileNames.SEGMENTS_GEN)) {
-        if (file.equals(IndexFileNames.SEGMENTS)) {
-          // Pre lock-less commits:
-          if (max == -1) {
-            max = 0;
-          }
-        } else {
-          long v = Long.parseLong(file.substring(prefixLen), Character.MAX_RADIX);
-          if (v > max) {
-            max = v;
-          }
+        long gen = generationFromSegmentsFileName(file);
+        if (gen > max) {
+          max = gen;
         }
       }
     }
@@ -152,6 +145,22 @@
   }
 
   /**
+   * Parse the generation off the segments file name and
+   * return it.
+   */
+  public static long generationFromSegmentsFileName(String fileName) {
+    if (fileName.equals(IndexFileNames.SEGMENTS)) {
+      return 0;
+    } else if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+      return Long.parseLong(fileName.substring(1+IndexFileNames.SEGMENTS.length()),
+                            Character.MAX_RADIX);
+    } else {
+      throw new IllegalArgumentException("fileName \"" + fileName + "\" is not a segments file");
+    }
+  }
+
+
+  /**
    * Get the next segments_N filename that will be written.
    */
   public String getNextSegmentFileName() {
@@ -181,12 +190,8 @@
 
     IndexInput input = directory.openInput(segmentFileName);
 
-    if (segmentFileName.equals(IndexFileNames.SEGMENTS)) {
-      generation = 0;
-    } else {
-      generation = Long.parseLong(segmentFileName.substring(1+IndexFileNames.SEGMENTS.length()),
-                                  Character.MAX_RADIX);
-    }
+    generation = generationFromSegmentsFileName(segmentFileName);
+
     lastGeneration = generation;
 
     try {
@@ -255,6 +260,8 @@
 
     IndexOutput output = directory.createOutput(segmentFileName);
 
+    boolean success = false;
+
     try {
       output.writeInt(FORMAT_SINGLE_NORM_FILE); // write FORMAT
       output.writeLong(++version); // every write changes
@@ -266,7 +273,16 @@
       }         
     }
     finally {
-      output.close();
+      try {
+        output.close();
+        success = true;
+      } finally {
+        if (!success) {
+          // Try not to leave a truncated segments_N file in
+          // the index:
+          directory.deleteFile(segmentFileName);
+        }
+      }
     }
 
     try {
@@ -304,6 +320,9 @@
    */
   public long getVersion() {
     return version;
+  }
+  public long getGeneration() {
+    return generation;
   }
 
   /**

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/SegmentReader.java Tue Mar 13 02:06:22 2007
@@ -77,15 +77,6 @@
 
     private void reWrite(SegmentInfo si) throws IOException {
       // NOTE: norms are re-written in regular directory, not cfs
-
-      String oldFileName = si.getNormFileName(this.number);
-      if (oldFileName != null && !oldFileName.endsWith("." + IndexFileNames.NORMS_EXTENSION)) {
-        // Mark this file for deletion.  Note that we don't
-        // actually try to delete it until the new segments files is
-        // successfully written:
-        deleter.addPendingFile(oldFileName);
-      }
-
       si.advanceNormGen(this.number);
       IndexOutput out = directory().createOutput(si.getNormFileName(this.number));
       try {
@@ -227,14 +218,6 @@
 
   protected void doCommit() throws IOException {
     if (deletedDocsDirty) {               // re-write deleted
-      String oldDelFileName = si.getDelFileName();
-      if (oldDelFileName != null) {
-        // Mark this file for deletion.  Note that we don't
-        // actually try to delete it until the new segments files is
-        // successfully written:
-        deleter.addPendingFile(oldDelFileName);
-      }
-
       si.advanceDelGen();
 
       // We can write directly to the actual name (vs to a
@@ -243,13 +226,6 @@
       deletedDocs.write(directory(), si.getDelFileName());
     }
     if (undeleteAll && si.hasDeletions()) {
-      String oldDelFileName = si.getDelFileName();
-      if (oldDelFileName != null) {
-        // Mark this file for deletion.  Note that we don't
-        // actually try to delete it until the new segments files is
-        // successfully written:
-        deleter.addPendingFile(oldDelFileName);
-      }
       si.clearDelGen();
     }
     if (normsDirty) {               // re-write norms
@@ -320,37 +296,7 @@
   }
 
   Vector files() throws IOException {
-    Vector files = new Vector(16);
-
-    if (si.getUseCompoundFile()) {
-      String name = segment + ".cfs";
-      if (directory().fileExists(name)) {
-        files.addElement(name);
-      }
-    } else {
-      for (int i = 0; i < IndexFileNames.INDEX_EXTENSIONS.length; i++) {
-        String name = segment + "." + IndexFileNames.INDEX_EXTENSIONS[i];
-        if (directory().fileExists(name))
-          files.addElement(name);
-      }
-    }
-
-    if (si.hasDeletions()) {
-      files.addElement(si.getDelFileName());
-    }
-
-    boolean addedNrm = false;
-    for (int i = 0; i < fieldInfos.size(); i++) {
-      String name = si.getNormFileName(i);
-      if (name != null && directory().fileExists(name)) {
-        if (name.endsWith("." + IndexFileNames.NORMS_EXTENSION)) {
-          if (addedNrm) continue; // add .nrm just once
-          addedNrm = true;
-        }
-        files.addElement(name);
-      }
-    }
-    return files;
+    return new Vector(si.files());
   }
 
   public TermEnum terms() {

Modified: lucene/java/trunk/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/site/src/documentation/content/xdocs/fileformats.xml?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/java/trunk/src/site/src/documentation/content/xdocs/fileformats.xml Tue Mar 13 02:06:22 2007
@@ -771,7 +771,9 @@
                     generation is the active one (when older
                     segments_N files are present it's because they
                     temporarily cannot be deleted, or, a writer is in
-                    the process of committing). This file lists each
+                    the process of committing, or a custom
+                    <a href="http://lucene.apache.org/java/docs/api/org/apache/lucene/index/IndexDeletionPolicy.html">IndexDeletionPolicy</a>
+		    is in use). This file lists each
                     segment by name, has details about the separate
                     norms and deletion files, and also contains the
                     size of each segment.

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Mar 13 02:06:22 2007
@@ -21,7 +21,7 @@
 import java.util.zip.*;
 
 /*
-  Verify we can read the pre-XXX file format, do searches
+  Verify we can read the pre-2.1 file format, do searches
   against it, and add documents to it.
 */
 
@@ -104,7 +104,11 @@
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
-      changeIndexNoAdds(oldNames[i]);
+      changeIndexNoAdds(oldNames[i], true);
+      rmDir(oldNames[i]);
+
+      unzip(dirName, oldNames[i]);
+      changeIndexNoAdds(oldNames[i], false);
       rmDir(oldNames[i]);
     }
   }
@@ -114,7 +118,11 @@
     for(int i=0;i<oldNames.length;i++) {
       String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
       unzip(dirName, oldNames[i]);
-      changeIndexWithAdds(oldNames[i]);
+      changeIndexWithAdds(oldNames[i], true);
+      rmDir(oldNames[i]);
+
+      unzip(dirName, oldNames[i]);
+      changeIndexWithAdds(oldNames[i], false);
       rmDir(oldNames[i]);
     }
   }
@@ -141,13 +149,14 @@
 
   /* Open pre-lockless index, add docs, do a delete &
    * setNorm, and search */
-  public void changeIndexWithAdds(String dirName) throws IOException {
+  public void changeIndexWithAdds(String dirName, boolean autoCommit) throws IOException {
 
     dirName = fullDir(dirName);
 
     Directory dir = FSDirectory.getDirectory(dirName);
+
     // open writer
-    IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+    IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
 
     // add 10 docs
     for(int i=0;i<10;i++) {
@@ -166,7 +175,7 @@
     assertEquals("wrong first document", "21", d.get("id"));
     searcher.close();
 
-    // make sure we can do another delete & another setNorm against this
+    // make sure we can do delete & setNorm against this
     // pre-lockless segment:
     IndexReader reader = IndexReader.open(dir);
     Term searchTerm = new Term("id", "6");
@@ -175,7 +184,7 @@
     reader.setNorm(22, "content", (float) 2.0);
     reader.close();
 
-    // make sure 2nd delete & 2nd norm "took":
+    // make sure they "took":
     searcher = new IndexSearcher(dir);
     hits = searcher.search(new TermQuery(new Term("content", "aaa")));
     assertEquals("wrong number of hits", 43, hits.length());
@@ -184,7 +193,7 @@
     searcher.close();
 
     // optimize
-    writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+    writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
     writer.optimize();
     writer.close();
 
@@ -200,7 +209,7 @@
 
   /* Open pre-lockless index, add docs, do a delete &
    * setNorm, and search */
-  public void changeIndexNoAdds(String dirName) throws IOException {
+  public void changeIndexNoAdds(String dirName, boolean autoCommit) throws IOException {
 
     dirName = fullDir(dirName);
 
@@ -214,7 +223,7 @@
     assertEquals("wrong first document", "21", d.get("id"));
     searcher.close();
 
-    // make sure we can do another delete & another setNorm against this
+    // make sure we can do a delete & setNorm against this
     // pre-lockless segment:
     IndexReader reader = IndexReader.open(dir);
     Term searchTerm = new Term("id", "6");
@@ -223,7 +232,7 @@
     reader.setNorm(22, "content", (float) 2.0);
     reader.close();
 
-    // make sure 2nd delete & 2nd norm "took":
+    // make sure they "took":
     searcher = new IndexSearcher(dir);
     hits = searcher.search(new TermQuery(new Term("content", "aaa")));
     assertEquals("wrong number of hits", 33, hits.length());
@@ -232,7 +241,7 @@
     searcher.close();
 
     // optimize
-    IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+    IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
     writer.optimize();
     writer.close();
 
@@ -273,66 +282,78 @@
 
   /* Verifies that the expected file names were produced */
 
-  // disable until hardcoded file names are fixes:
   public void testExactFileNames() throws IOException {
 
-    String outputDir = "lucene.backwardscompat0.index";
-    Directory dir = FSDirectory.getDirectory(fullDir(outputDir));
-    IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
-    for(int i=0;i<35;i++) {
-      addDoc(writer, i);
-    }
-    assertEquals("wrong doc count", 35, writer.docCount());
-    writer.close();
+    for(int pass=0;pass<2;pass++) {
 
-    // Delete one doc so we get a .del file:
-    IndexReader reader = IndexReader.open(dir);
-    Term searchTerm = new Term("id", "7");
-    int delCount = reader.deleteDocuments(searchTerm);
-    assertEquals("didn't delete the right number of documents", 1, delCount);
+      String outputDir = "lucene.backwardscompat0.index";
 
-    // Set one norm so we get a .s0 file:
-    reader.setNorm(21, "content", (float) 1.5);
-    reader.close();
+      try {
+        Directory dir = FSDirectory.getDirectory(fullDir(outputDir));
 
-    // The numbering of fields can vary depending on which
-    // JRE is in use.  On some JREs we see content bound to
-    // field 0; on others, field 1.  So, here we have to
-    // figure out which field number corresponds to
-    // "content", and then set our expected file names below
-    // accordingly:
-    CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
-    FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
-    int contentFieldIndex = -1;
-    for(int i=0;i<fieldInfos.size();i++) {
-      FieldInfo fi = fieldInfos.fieldInfo(i);
-      if (fi.name.equals("content")) {
-        contentFieldIndex = i;
-        break;
+        boolean autoCommit = 0 == pass;
+      
+        IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+        //IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        for(int i=0;i<35;i++) {
+          addDoc(writer, i);
+        }
+        assertEquals("wrong doc count", 35, writer.docCount());
+        writer.close();
+
+        // Delete one doc so we get a .del file:
+        IndexReader reader = IndexReader.open(dir);
+        Term searchTerm = new Term("id", "7");
+        int delCount = reader.deleteDocuments(searchTerm);
+        assertEquals("didn't delete the right number of documents", 1, delCount);
+
+        // Set one norm so we get a .s0 file:
+        reader.setNorm(21, "content", (float) 1.5);
+        reader.close();
+
+        // The numbering of fields can vary depending on which
+        // JRE is in use.  On some JREs we see content bound to
+        // field 0; on others, field 1.  So, here we have to
+        // figure out which field number corresponds to
+        // "content", and then set our expected file names below
+        // accordingly:
+        CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs");
+        FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm");
+        int contentFieldIndex = -1;
+        for(int i=0;i<fieldInfos.size();i++) {
+          FieldInfo fi = fieldInfos.fieldInfo(i);
+          if (fi.name.equals("content")) {
+            contentFieldIndex = i;
+            break;
+          }
+        }
+        cfsReader.close();
+        assertTrue("could not locate the 'content' field number in the _2.cfs segment", contentFieldIndex != -1);
+
+        // Now verify file names:
+        String[] expected = {"_0.cfs",
+                             "_0_1.del",
+                             "_1.cfs",
+                             "_2.cfs",
+                             "_2_1.s" + contentFieldIndex,
+                             "_3.cfs",
+                             "segments_a",
+                             "segments.gen"};
+        if (!autoCommit) {
+          expected[6] = "segments_3";
+        }
+
+        String[] actual = dir.list();
+        Arrays.sort(expected);
+        Arrays.sort(actual);
+        if (!Arrays.equals(expected, actual)) {
+          fail("incorrect filenames in index: expected:\n    " + asString(expected) + "\n  actual:\n    " + asString(actual));
+        }
+        dir.close();
+      } finally {
+        rmDir(outputDir);
       }
     }
-    cfsReader.close();
-    assertTrue("could not locate the 'content' field number in the _2.cfs segment", contentFieldIndex != -1);
-
-    // Now verify file names:
-    String[] expected = {"_0.cfs",
-                         "_0_1.del",
-                         "_1.cfs",
-                         "_2.cfs",
-                         "_2_1.s" + contentFieldIndex,
-                         "_3.cfs",
-                         "segments_a",
-                         "segments.gen"};
-
-    String[] actual = dir.list();
-    Arrays.sort(expected);
-    Arrays.sort(actual);
-    if (!Arrays.equals(expected, actual)) {
-      fail("incorrect filenames in index: expected:\n    " + asString(expected) + "\n  actual:\n    " + asString(actual));
-    }
-    dir.close();
-
-    rmDir(outputDir);
   }
 
   private String asString(String[] l) {

Added: lucene/java/trunk/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestDeletionPolicy.java?view=auto&rev=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestDeletionPolicy.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestDeletionPolicy.java Tue Mar 13 02:06:22 2007
@@ -0,0 +1,618 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import java.util.List;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.HashSet;
+
+/*
+  Verify we can read the pre-2.1 file format, do searches
+  against it, and add documents to it.
+*/
+
+public class TestDeletionPolicy extends TestCase
+{
+  private void verifyCommitOrder(List commits) {
+    long last = SegmentInfos.generationFromSegmentsFileName(((IndexCommitPoint) commits.get(0)).getSegmentsFileName());
+    for(int i=1;i<commits.size();i++) {
+      long now = SegmentInfos.generationFromSegmentsFileName(((IndexCommitPoint) commits.get(i)).getSegmentsFileName());
+      assertTrue("SegmentInfos commits are out-of-order", now > last);
+      last = now;
+    }
+  }
+
+  class KeepAllDeletionPolicy implements IndexDeletionPolicy {
+    int numOnInit;
+    int numOnCommit;
+    public void onInit(List commits) {
+      verifyCommitOrder(commits);
+      numOnInit++;
+    }
+    public void onCommit(List commits) {
+      verifyCommitOrder(commits);
+      numOnCommit++;
+    }
+  }
+
+  /**
+   * This is useful for adding to a big index w/ autoCommit
+   * false when you know readers are not using it.
+   */
+  class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy {
+    int numOnInit;
+    int numOnCommit;
+    public void onInit(List commits) {
+      verifyCommitOrder(commits);
+      numOnInit++;
+      // On init, delete all commit points:
+      Iterator it = commits.iterator();
+      while(it.hasNext()) {
+        ((IndexCommitPoint) it.next()).delete();
+      }
+    }
+    public void onCommit(List commits) {
+      verifyCommitOrder(commits);
+      int size = commits.size();
+      // Delete all but last one:
+      for(int i=0;i<size-1;i++) {
+        ((IndexCommitPoint) commits.get(i)).delete();
+      }
+      numOnCommit++;
+    }
+  }
+
+  class KeepLastNDeletionPolicy implements IndexDeletionPolicy {
+    int numOnInit;
+    int numOnCommit;
+    int numToKeep;
+    int numDelete;
+    Set seen = new HashSet();
+
+    public KeepLastNDeletionPolicy(int numToKeep) {
+      this.numToKeep = numToKeep;
+    }
+
+    public void onInit(List commits) {
+      verifyCommitOrder(commits);
+      numOnInit++;
+      // do no deletions on init
+      doDeletes(commits, false);
+    }
+
+    public void onCommit(List commits) {
+      verifyCommitOrder(commits);
+      doDeletes(commits, true);
+    }
+    
+    private void doDeletes(List commits, boolean isCommit) {
+
+      // Assert that we really are only called for each new
+      // commit:
+      if (isCommit) {
+        String fileName = ((IndexCommitPoint) commits.get(commits.size()-1)).getSegmentsFileName();
+        if (seen.contains(fileName)) {
+          throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName);
+        }
+        seen.add(fileName);
+        numOnCommit++;
+      }
+      int size = commits.size();
+      for(int i=0;i<size-numToKeep;i++) {
+        ((IndexCommitPoint) commits.get(i)).delete();
+        numDelete++;
+      }
+    }
+  }
+
+  /*
+   * Delete a commit only when it has been obsoleted by N
+   * seconds.
+   */
+  class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy {
+
+    Directory dir;
+    double expirationTimeSeconds;
+    int numDelete;
+
+    public ExpirationTimeDeletionPolicy(Directory dir, double seconds) {
+      this.dir = dir;
+      this.expirationTimeSeconds = seconds;
+    }
+
+    public void onInit(List commits) throws IOException {
+      verifyCommitOrder(commits);
+      onCommit(commits);
+    }
+
+    public void onCommit(List commits) throws IOException {
+      verifyCommitOrder(commits);
+
+      IndexCommitPoint lastCommit = (IndexCommitPoint) commits.get(commits.size()-1);
+
+      // Any commit older than expireTime should be deleted:
+      double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds;
+
+      Iterator it = commits.iterator();
+
+      while(it.hasNext()) {
+        IndexCommitPoint commit = (IndexCommitPoint) it.next();
+        double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0;
+        if (commit != lastCommit && modTime < expireTime) {
+          commit.delete();
+          numDelete += 1;
+        }
+      }
+    }
+  }
+
+  /*
+   * Test "by time expiration" deletion policy:
+   */
+  public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {
+
+    final double SECONDS = 2.0;
+
+    boolean autoCommit = false;
+    boolean useCompoundFile = true;
+
+    Directory dir = new RAMDirectory();
+    ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
+    IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+    writer.setUseCompoundFile(useCompoundFile);
+    writer.close();
+
+    for(int i=0;i<7;i++) {
+      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      for(int j=0;j<17;j++) {
+        addDoc(writer);
+      }
+      writer.close();
+
+      // Make sure to sleep long enough so that some commit
+      // points will be deleted:
+      Thread.sleep((int) (1000.0*(SECONDS/5.0)));
+    }
+
+    // First, make sure the policy in fact deleted something:
+    assertTrue("no commits were deleted", policy.numDelete > 0);
+
+    // Then simplistic check: just verify that the
+    // segments_N's that still exist are in fact within SECONDS
+    // seconds of the last one's mod time, and, that I can
+    // open a reader on each:
+    long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+    
+    String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                            "",
+                                                            gen);
+    long newestModTime = dir.fileModified(fileName);
+      
+    while(gen > 0) {
+      try {
+        IndexReader reader = IndexReader.open(dir);
+        reader.close();
+        fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                         "",
+                                                         gen);
+        long modTime = dir.fileModified(fileName);
+        assertTrue("commit point was older than " + SECONDS + " seconds but did not get deleted", newestModTime - modTime < (SECONDS*1000));
+      } catch (IOException e) {
+        // OK
+        break;
+      }
+      
+      dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+      gen--;
+    }
+
+    dir.close();
+  }
+
+  /*
+   * Test a silly deletion policy that keeps all commits around.
+   */
+  public void testKeepAllDeletionPolicy() throws IOException {
+
+    for(int pass=0;pass<4;pass++) {
+
+      boolean autoCommit = pass < 2;
+      boolean useCompoundFile = (pass % 2) > 0;
+
+      KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
+
+      Directory dir = new RAMDirectory();
+
+      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      for(int i=0;i<107;i++) {
+        addDoc(writer);
+      }
+      writer.close();
+
+      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      writer.optimize();
+      writer.close();
+
+      assertEquals(2, policy.numOnInit);
+      if (autoCommit) {
+        assertTrue(policy.numOnCommit > 2);
+      } else {
+        // If we are not auto committing then there should
+        // be exactly 2 commits (one per close above):
+        assertEquals(2, policy.numOnCommit);
+      }
+
+      // Simplistic check: just verify all segments_N's still
+      // exist, and, I can open a reader on each:
+      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+      while(gen > 0) {
+        IndexReader reader = IndexReader.open(dir);
+        reader.close();
+        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+        gen--;
+
+        if (gen > 0) {
+          // Now that we've removed a commit point, which
+          // should have orphan'd at least one index file.
+          // Open & close a writer and assert that it
+          // actually removed something:
+          int preCount = dir.list().length;
+          writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false, policy);
+          writer.close();
+          int postCount = dir.list().length;
+          assertTrue(postCount < preCount);
+        }
+      }
+
+      dir.close();
+    }
+  }
+
+  /* Test keeping NO commit points.  This is a viable and
+   * useful case eg where you want to build a big index with
+   * autoCommit false and you know there are no readers.
+   */
+  public void testKeepNoneOnInitDeletionPolicy() throws IOException {
+
+    for(int pass=0;pass<4;pass++) {
+
+      boolean autoCommit = pass < 2;
+      boolean useCompoundFile = (pass % 2) > 0;
+
+      KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();
+
+      Directory dir = new RAMDirectory();
+
+      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      for(int i=0;i<107;i++) {
+        addDoc(writer);
+      }
+      writer.close();
+
+      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      writer.optimize();
+      writer.close();
+
+      assertEquals(2, policy.numOnInit);
+      if (autoCommit) {
+        assertTrue(policy.numOnCommit > 2);
+      } else {
+        // If we are not auto committing then there should
+        // be exactly 2 commits (one per close above):
+        assertEquals(2, policy.numOnCommit);
+      }
+
+      // Simplistic check: just verify the index is in fact
+      // readable:
+      IndexReader reader = IndexReader.open(dir);
+      reader.close();
+
+      dir.close();
+    }
+  }
+
+  /*
+   * Test a deletion policy that keeps last N commits.
+   */
+  public void testKeepLastNDeletionPolicy() throws IOException {
+
+    final int N = 5;
+
+    for(int pass=0;pass<4;pass++) {
+
+      boolean autoCommit = pass < 2;
+      boolean useCompoundFile = (pass % 2) > 0;
+
+      Directory dir = new RAMDirectory();
+
+      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+      for(int j=0;j<N+1;j++) {
+        IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+        writer.setUseCompoundFile(useCompoundFile);
+        for(int i=0;i<17;i++) {
+          addDoc(writer);
+        }
+        writer.optimize();
+        writer.close();
+      }
+
+      assertTrue(policy.numDelete > 0);
+      assertEquals(N+1, policy.numOnInit);
+      if (autoCommit) {
+        assertTrue(policy.numOnCommit > 1);
+      } else {
+        assertEquals(N+1, policy.numOnCommit);
+      }
+
+      // Simplistic check: just verify only the past N segments_N's still
+      // exist, and, I can open a reader on each:
+      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+      for(int i=0;i<N+1;i++) {
+        try {
+          IndexReader reader = IndexReader.open(dir);
+          reader.close();
+          if (i == N) {
+            fail("should have failed on commits prior to last " + N);
+          }
+        } catch (IOException e) {
+          if (i != N) {
+            throw e;
+          }
+        }
+        if (i < N) {
+          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+        }
+        gen--;
+      }
+
+      dir.close();
+    }
+  }
+
+  /*
+   * Test a deletion policy that keeps last N commits
+   * around, with reader doing deletes.
+   */
+  public void testKeepLastNDeletionPolicyWithReader() throws IOException {
+
+    final int N = 10;
+
+    for(int pass=0;pass<4;pass++) {
+
+      boolean autoCommit = pass < 2;
+      boolean useCompoundFile = (pass % 2) > 0;
+
+      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+      Directory dir = new RAMDirectory();
+      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      writer.close();
+      Term searchTerm = new Term("content", "aaa");        
+      Query query = new TermQuery(searchTerm);
+
+      for(int i=0;i<N+1;i++) {
+        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+        writer.setUseCompoundFile(useCompoundFile);
+        for(int j=0;j<17;j++) {
+          addDoc(writer);
+        }
+        // this is a commit when autoCommit=false:
+        writer.close();
+        IndexReader reader = IndexReader.open(dir, policy);
+        reader.deleteDocument(3*i+1);
+        reader.setNorm(4*i+1, "content", 2.0F);
+        IndexSearcher searcher = new IndexSearcher(reader);
+        Hits hits = searcher.search(query);
+        assertEquals(16*(1+i), hits.length());
+        // this is a commit when autoCommit=false:
+        reader.close();
+        searcher.close();
+      }
+      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      writer.optimize();
+      // this is a commit when autoCommit=false:
+      writer.close();
+
+      assertEquals(2*(N+2), policy.numOnInit);
+      if (autoCommit) {
+        assertTrue(policy.numOnCommit > 2*(N+2)-1);
+      } else {
+        assertEquals(2*(N+2)-1, policy.numOnCommit);
+      }
+
+      IndexSearcher searcher = new IndexSearcher(dir);
+      Hits hits = searcher.search(query);
+      assertEquals(176, hits.length());
+
+      // Simplistic check: just verify only the past N segments_N's still
+      // exist, and, I can open a reader on each:
+      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+
+      int expectedCount = 176;
+
+      for(int i=0;i<N+1;i++) {
+        try {
+          IndexReader reader = IndexReader.open(dir);
+
+          // Work backwards in commits on what the expected
+          // count should be.  Only check this in the
+          // autoCommit false case:
+          if (!autoCommit) {
+            searcher = new IndexSearcher(reader);
+            hits = searcher.search(query);
+            if (i > 1) {
+              if (i % 2 == 0) {
+                expectedCount += 1;
+              } else {
+                expectedCount -= 17;
+              }
+            }
+            assertEquals(expectedCount, hits.length());
+            searcher.close();
+          }
+          reader.close();
+          if (i == N) {
+            fail("should have failed on commits before last 5");
+          }
+        } catch (IOException e) {
+          if (i != N) {
+            throw e;
+          }
+        }
+        if (i < N) {
+          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+        }
+        gen--;
+      }
+      
+      dir.close();
+    }
+  }
+
+  /*
+   * Test a deletion policy that keeps last N commits
+   * around, through creates.
+   */
+  public void testKeepLastNDeletionPolicyWithCreates() throws IOException {
+
+    final int N = 10;
+
+    for(int pass=0;pass<4;pass++) {
+
+      boolean autoCommit = pass < 2;
+      boolean useCompoundFile = (pass % 2) > 0;
+
+      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+      Directory dir = new RAMDirectory();
+      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+      writer.setUseCompoundFile(useCompoundFile);
+      writer.close();
+      Term searchTerm = new Term("content", "aaa");        
+      Query query = new TermQuery(searchTerm);
+
+      for(int i=0;i<N+1;i++) {
+
+        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
+        writer.setUseCompoundFile(useCompoundFile);
+        for(int j=0;j<17;j++) {
+          addDoc(writer);
+        }
+        // this is a commit when autoCommit=false:
+        writer.close();
+        IndexReader reader = IndexReader.open(dir, policy);
+        reader.deleteDocument(3);
+        reader.setNorm(5, "content", 2.0F);
+        IndexSearcher searcher = new IndexSearcher(reader);
+        Hits hits = searcher.search(query);
+        assertEquals(16, hits.length());
+        // this is a commit when autoCommit=false:
+        reader.close();
+        searcher.close();
+
+        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
+        // This will not commit: there are no changes
+        // pending because we opened for "create":
+        writer.close();
+      }
+
+      assertEquals(1+3*(N+1), policy.numOnInit);
+      if (autoCommit) {
+        assertTrue(policy.numOnCommit > 3*(N+1)-1);
+      } else {
+        assertEquals(2*(N+1), policy.numOnCommit);
+      }
+
+      IndexSearcher searcher = new IndexSearcher(dir);
+      Hits hits = searcher.search(query);
+      assertEquals(0, hits.length());
+
+      // Simplistic check: just verify only the past N segments_N's still
+      // exist, and, I can open a reader on each:
+      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+
+      int expectedCount = 0;
+
+      for(int i=0;i<N+1;i++) {
+        try {
+          IndexReader reader = IndexReader.open(dir);
+
+          // Work backwards in commits on what the expected
+          // count should be.  Only check this in the
+          // autoCommit false case:
+          if (!autoCommit) {
+            searcher = new IndexSearcher(reader);
+            hits = searcher.search(query);
+            assertEquals(expectedCount, hits.length());
+            searcher.close();
+            if (expectedCount == 0) {
+              expectedCount = 16;
+            } else if (expectedCount == 16) {
+              expectedCount = 17;
+            } else if (expectedCount == 17) {
+              expectedCount = 0;
+            }
+          }
+          reader.close();
+          if (i == N) {
+            fail("should have failed on commits before last " + N);
+          }
+        } catch (IOException e) {
+          if (i != N) {
+            throw e;
+          }
+        }
+        if (i < N) {
+          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+        }
+        gen--;
+      }
+      
+      dir.close();
+    }
+  }
+
+  private void addDoc(IndexWriter writer) throws IOException
+  {
+    Document doc = new Document();
+    doc.add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
+    writer.addDocument(doc);
+  }
+}

Propchange: lucene/java/trunk/src/test/org/apache/lucene/index/TestDeletionPolicy.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Tue Mar 13 02:06:22 2007
@@ -173,6 +173,8 @@
       out.writeBytes(b, len);
       remainder -= len;
     }
+    in.close();
+    out.close();
   }
 
   private void addDoc(IndexWriter writer, int id) throws IOException

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexReader.java Tue Mar 13 02:06:22 2007
@@ -802,9 +802,7 @@
           String[] startFiles = dir.list();
           SegmentInfos infos = new SegmentInfos();
           infos.read(dir);
-          IndexFileDeleter d = new IndexFileDeleter(infos, dir);
-          d.findDeletableFiles();
-          d.deleteFiles();
+          IndexFileDeleter d = new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
           String[] endFiles = dir.list();
 
           Arrays.sort(startFiles);

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Mar 13 02:06:22 2007
@@ -71,7 +71,7 @@
         reader.close();
 
         // optimize the index and check that the new doc count is correct
-        writer = new IndexWriter(dir, new WhitespaceAnalyzer());
+        writer = new IndexWriter(dir, true, new WhitespaceAnalyzer());
         writer.optimize();
         assertEquals(60, writer.docCount());
         writer.close();
@@ -163,7 +163,7 @@
       // addIndexes will certainly run out of space &
       // fail.  Verify that when this happens, index is
       // not corrupt and index in fact has added no
-      // documents.  Then, we increase disk space by 1000
+      // documents.  Then, we increase disk space by 2000
       // bytes each iteration.  At some point there is
       // enough free disk space and addIndexes should
       // succeed and index should show all documents were
@@ -178,11 +178,14 @@
         startDiskUsage += startDir.fileLength(files[i]);
       }
 
-      for(int method=0;method<3;method++) {
+      for(int iter=0;iter<6;iter++) {
 
         // Start with 100 bytes more than we are currently using:
         long diskFree = diskUsage+100;
 
+        boolean autoCommit = iter % 2 == 0;
+        int method = iter/2;
+
         boolean success = false;
         boolean done = false;
 
@@ -195,7 +198,7 @@
           methodName = "addIndexesNoOptimize(Directory[])";
         }
 
-        String testName = "disk full test for method " + methodName + " with disk full at " + diskFree + " bytes";
+        String testName = "disk full test for method " + methodName + " with disk full at " + diskFree + " bytes with autoCommit = " + autoCommit;
 
         int cycleCount = 0;
 
@@ -205,7 +208,7 @@
 
           // Make a new dir that will enforce disk usage:
           MockRAMDirectory dir = new MockRAMDirectory(startDir);
-          writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+          writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
           IOException err = null;
 
           for(int x=0;x<2;x++) {
@@ -285,38 +288,27 @@
               }
             }
 
-            // Whether we succeeded or failed, check that all
-            // un-referenced files were in fact deleted (ie,
-            // we did not create garbage).  Just create a
-            // new IndexFileDeleter, have it delete
-            // unreferenced files, then verify that in fact
-            // no files were deleted:
-            String[] startFiles = dir.list();
-            SegmentInfos infos = new SegmentInfos();
-            infos.read(dir);
-            IndexFileDeleter d = new IndexFileDeleter(infos, dir);
-            d.findDeletableFiles();
-            d.deleteFiles();
-            String[] endFiles = dir.list();
-
-            Arrays.sort(startFiles);
-            Arrays.sort(endFiles);
-
-            /*
-              for(int i=0;i<startFiles.length;i++) {
-              System.out.println("  " + i + ": " + startFiles[i]);
-              }
-            */
+            if (autoCommit) {
+
+              // Whether we succeeded or failed, check that
+              // all un-referenced files were in fact
+              // deleted (ie, we did not create garbage).
+              // Only check this when autoCommit is true:
+              // when it's false, it's expected that there
+              // are unreferenced files (ie they won't be
+              // referenced until the "commit on close").
+              // Just create a new IndexFileDeleter, have it
+              // delete unreferenced files, then verify that
+              // in fact no files were deleted:
 
-            if (!Arrays.equals(startFiles, endFiles)) {
               String successStr;
               if (success) {
                 successStr = "success";
               } else {
                 successStr = "IOException";
-                err.printStackTrace();
               }
-              fail(methodName + " failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n    " + arrayToString(startFiles) + "\n  after delete:\n    " + arrayToString(endFiles));
+              String message = methodName + " failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes)";
+              assertNoUnreferencedFiles(dir, message);
             }
 
             if (debug) {
@@ -335,8 +327,10 @@
             }
             int result = reader.docFreq(searchTerm);
             if (success) {
-              if (result != END_COUNT) {
+              if (autoCommit && result != END_COUNT) {
                 fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + END_COUNT);
+              } else if (!autoCommit && result != START_COUNT) {
+                fail(testName + ": method did not throw exception but docFreq('aaa') is " + result + " instead of expected " + START_COUNT + " [autoCommit = false]");
               }
             } else {
               // On hitting exception we still may have added
@@ -374,31 +368,107 @@
               System.out.println("  count is " + result);
             }
 
-            if (result == END_COUNT) {
+            if (done || result == END_COUNT) {
               break;
             }
           }
 
-          // Javadocs state that temp free Directory space
-          // required is at most 2X total input size of
-          // indices so let's make sure:
-          assertTrue("max free Directory space required exceeded 1X the total input index sizes during " + methodName +
-                     ": max temp usage = " + (dir.getMaxUsedSizeInBytes()-startDiskUsage) + " bytes; " +
-                     "starting disk usage = " + startDiskUsage + " bytes; " +
-                     "input index disk usage = " + inputDiskUsage + " bytes",
-                     (dir.getMaxUsedSizeInBytes()-startDiskUsage) < 2*(startDiskUsage + inputDiskUsage));
+          if (debug) {
+            System.out.println("  start disk = " + startDiskUsage + "; input disk = " + inputDiskUsage + "; max used = " + dir.getMaxUsedSizeInBytes());
+          }
+
+          if (done) {
+            // Javadocs state that temp free Directory space
+            // required is at most 2X total input size of
+            // indices so let's make sure:
+            assertTrue("max free Directory space required exceeded 1X the total input index sizes during " + methodName +
+                       ": max temp usage = " + (dir.getMaxUsedSizeInBytes()-startDiskUsage) + " bytes; " +
+                       "starting disk usage = " + startDiskUsage + " bytes; " +
+                       "input index disk usage = " + inputDiskUsage + " bytes",
+                       (dir.getMaxUsedSizeInBytes()-startDiskUsage) < 2*(startDiskUsage + inputDiskUsage));
+          }
 
           writer.close();
           dir.close();
 
-          // Try again with 1000 more bytes of free space:
-          diskFree += 1000;
+          // Try again with 2000 more bytes of free space:
+          diskFree += 2000;
         }
       }
 
       startDir.close();
     }
 
+    /*
+     * Make sure IndexWriter cleans up on hitting a disk
+     * full exception in addDocument.
+     */
+    public void testAddDocumentOnDiskFull() throws IOException {
+
+      for(int pass=0;pass<3;pass++) {
+        boolean autoCommit = pass == 0;
+        boolean doAbort = pass == 2;
+        long diskFree = 200;
+        while(true) {
+          MockRAMDirectory dir = new MockRAMDirectory();
+          dir.setMaxSizeInBytes(diskFree);
+          IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+          boolean hitError = false;
+          try {
+            for(int i=0;i<200;i++) {
+              addDoc(writer);
+            }
+          } catch (IOException e) {
+            // e.printStackTrace();
+            hitError = true;
+          }
+
+          if (hitError) {
+            if (doAbort) {
+              writer.abort();
+            } else {
+              try {
+                writer.close();
+              } catch (IOException e) {
+                // e.printStackTrace();
+                dir.setMaxSizeInBytes(0);
+                writer.close();
+              }
+            }
+
+            assertNoUnreferencedFiles(dir, "after disk full during addDocument with autoCommit=" + autoCommit);
+
+            // Make sure reader can open the index:
+            IndexReader.open(dir).close();
+
+            dir.close();
+
+            // Now try again w/ more space:
+            diskFree += 500;
+          } else {
+            dir.close();
+            break;
+          }
+        }
+      }
+    
+    }                                               
+
+    public void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
+      String[] startFiles = dir.list();
+      SegmentInfos infos = new SegmentInfos();
+      infos.read(dir);
+      IndexFileDeleter d = new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
+      String[] endFiles = dir.list();
+
+      Arrays.sort(startFiles);
+      Arrays.sort(endFiles);
+
+      if (!Arrays.equals(startFiles, endFiles)) {
+        fail(message + ": before delete:\n    " + arrayToString(startFiles) + "\n  after delete:\n    " + arrayToString(endFiles));
+      }
+    }
+
     /**
      * Make sure optimize doesn't use any more than 1X
      * starting index size as its temporary free space
@@ -692,6 +762,205 @@
         if (reader != null) {
           reader.close();
         }
+    }
+
+    /*
+     * Simple test for "commit on close": open writer with
+     * autoCommit=false, so it will only commit on close,
+     * then add a bunch of docs, making sure reader does not
+     * see these docs until writer is closed.
+     */
+    public void testCommitOnClose() throws IOException {
+        Directory dir = new RAMDirectory();      
+        IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+        for (int i = 0; i < 14; i++) {
+          addDoc(writer);
+        }
+        writer.close();
+
+        Term searchTerm = new Term("content", "aaa");        
+        IndexSearcher searcher = new IndexSearcher(dir);
+        Hits hits = searcher.search(new TermQuery(searchTerm));
+        assertEquals("first number of hits", 14, hits.length());
+        searcher.close();
+
+        IndexReader reader = IndexReader.open(dir);
+
+        writer = new IndexWriter(dir, false, new WhitespaceAnalyzer());
+        for(int i=0;i<3;i++) {
+          for(int j=0;j<11;j++) {
+            addDoc(writer);
+          }
+          searcher = new IndexSearcher(dir);
+          hits = searcher.search(new TermQuery(searchTerm));
+          assertEquals("reader incorrectly sees changes from writer with autoCommit disabled", 14, hits.length());
+          searcher.close();
+          assertTrue("reader should have still been current", reader.isCurrent());
+        }
+
+        // Now, close the writer:
+        writer.close();
+        assertFalse("reader should not be current now", reader.isCurrent());
+
+        searcher = new IndexSearcher(dir);
+        hits = searcher.search(new TermQuery(searchTerm));
+        assertEquals("reader did not see changes after writer was closed", 47, hits.length());
+        searcher.close();
+    }
+
+    /*
+     * Simple test for "commit on close": open writer with
+     * autoCommit=false, so it will only commit on close,
+     * then add a bunch of docs, making sure reader does not
+     * see them until writer has closed.  Then instead of
+     * closing the writer, call abort and verify reader sees
+     * nothing was added.  Then verify we can open the index
+     * and add docs to it.
+     */
+    public void testCommitOnCloseAbort() throws IOException {
+      Directory dir = new RAMDirectory();      
+      IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      for (int i = 0; i < 14; i++) {
+        addDoc(writer);
+      }
+      writer.close();
+
+      Term searchTerm = new Term("content", "aaa");        
+      IndexSearcher searcher = new IndexSearcher(dir);
+      Hits hits = searcher.search(new TermQuery(searchTerm));
+      assertEquals("first number of hits", 14, hits.length());
+      searcher.close();
+
+      writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false);
+      for(int j=0;j<17;j++) {
+        addDoc(writer);
+      }
+      // Delete all docs:
+      writer.deleteDocuments(searchTerm);
+
+      searcher = new IndexSearcher(dir);
+      hits = searcher.search(new TermQuery(searchTerm));
+      assertEquals("reader incorrectly sees changes from writer with autoCommit disabled", 14, hits.length());
+      searcher.close();
+
+      // Now, close the writer:
+      writer.abort();
+
+      assertNoUnreferencedFiles(dir, "unreferenced files remain after abort()");
+
+      searcher = new IndexSearcher(dir);
+      hits = searcher.search(new TermQuery(searchTerm));
+      assertEquals("saw changes after writer.abort", 14, hits.length());
+      searcher.close();
+          
+      // Now make sure we can re-open the index, add docs,
+      // and all is good:
+      writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false);
+      for(int i=0;i<12;i++) {
+        for(int j=0;j<17;j++) {
+          addDoc(writer);
+        }
+        searcher = new IndexSearcher(dir);
+        hits = searcher.search(new TermQuery(searchTerm));
+        assertEquals("reader incorrectly sees changes from writer with autoCommit disabled", 14, hits.length());
+        searcher.close();
+      }
+
+      writer.close();
+      searcher = new IndexSearcher(dir);
+      hits = searcher.search(new TermQuery(searchTerm));
+      assertEquals("didn't see changes after close", 218, hits.length());
+      searcher.close();
+
+      dir.close();
+    }
+
+    /*
+     * Verify that a writer with "commit on close" indeed
+     * cleans up the temp segments created after opening
+     * that are not referenced by the starting segments
+     * file.  We check this by using MockRAMDirectory to
+     * measure max temp disk space used.
+     */
+    public void testCommitOnCloseDiskUsage() throws IOException {
+      MockRAMDirectory dir = new MockRAMDirectory();      
+      IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      for(int j=0;j<30;j++) {
+        addDocWithIndex(writer, j);
+      }
+      writer.close();
+      dir.resetMaxUsedSizeInBytes();
+
+      long startDiskUsage = dir.getMaxUsedSizeInBytes();
+      writer  = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false);
+      for(int j=0;j<1470;j++) {
+        addDocWithIndex(writer, j);
+      }
+      long midDiskUsage = dir.getMaxUsedSizeInBytes();
+      dir.resetMaxUsedSizeInBytes();
+      writer.optimize();
+      writer.close();
+      long endDiskUsage = dir.getMaxUsedSizeInBytes();
+
+      // Ending index is 50X as large as starting index; due
+      // to 2X disk usage normally we allow 100X max
+      // transient usage.  If something is wrong w/ deleter
+      // and it doesn't delete intermediate segments then it
+      // will exceed this 100X:
+      // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
+      assertTrue("writer used to much space while adding documents when autoCommit=false",     
+                 midDiskUsage < 100*startDiskUsage);
+      assertTrue("writer used to much space after close when autoCommit=false",     
+                 endDiskUsage < 100*startDiskUsage);
+    }
+
+
+    /*
+     * Verify that calling optimize when writer is open for
+     * "commit on close" works correctly both for abort()
+     * and close().
+     */
+    public void testCommitOnCloseOptimize() throws IOException {
+      RAMDirectory dir = new RAMDirectory();      
+      IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+      for(int j=0;j<17;j++) {
+        addDocWithIndex(writer, j);
+      }
+      writer.close();
+
+      writer  = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false);
+      writer.optimize();
+
+      // Open a reader before closing (commiting) the writer:
+      IndexReader reader = IndexReader.open(dir);
+
+      // Reader should see index as unoptimized at this
+      // point:
+      assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
+      reader.close();
+
+      // Abort the writer:
+      writer.abort();
+      assertNoUnreferencedFiles(dir, "aborted writer after optimize");
+
+      // Open a reader after aborting writer:
+      reader = IndexReader.open(dir);
+
+      // Reader should still see index as unoptimized:
+      assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
+      reader.close();
+
+      writer  = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false);
+      writer.optimize();
+      writer.close();
+      assertNoUnreferencedFiles(dir, "aborted writer after optimize");
+
+      // Open a reader after aborting writer:
+      reader = IndexReader.open(dir);
+
+      // Reader should still see index as unoptimized:
+      assertTrue("Reader incorrectly sees that the index is unoptimized", reader.isOptimized());
+      reader.close();
     }
 
     // Make sure that a Directory implementation that does

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue Mar 13 02:06:22 2007
@@ -25,175 +25,259 @@
         "Venice has lots of canals" };
     String[] text = { "Amsterdam", "Venice" };
 
-    Directory dir = new RAMDirectory();
-    IndexWriter modifier = new IndexWriter(dir,
-        new WhitespaceAnalyzer(), true);
-    modifier.setUseCompoundFile(true);
-    modifier.setMaxBufferedDeleteTerms(1);
-
-    for (int i = 0; i < keywords.length; i++) {
-      Document doc = new Document();
-      doc.add(new Field("id", keywords[i], Field.Store.YES,
-          Field.Index.UN_TOKENIZED));
-      doc.add(new Field("country", unindexed[i], Field.Store.YES,
-          Field.Index.NO));
-      doc.add(new Field("contents", unstored[i], Field.Store.NO,
-          Field.Index.TOKENIZED));
-      doc
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
+
+      Directory dir = new RAMDirectory();
+      IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                             new WhitespaceAnalyzer(), true);
+      modifier.setUseCompoundFile(true);
+      modifier.setMaxBufferedDeleteTerms(1);
+
+      for (int i = 0; i < keywords.length; i++) {
+        Document doc = new Document();
+        doc.add(new Field("id", keywords[i], Field.Store.YES,
+                          Field.Index.UN_TOKENIZED));
+        doc.add(new Field("country", unindexed[i], Field.Store.YES,
+                          Field.Index.NO));
+        doc.add(new Field("contents", unstored[i], Field.Store.NO,
+                          Field.Index.TOKENIZED));
+        doc
           .add(new Field("city", text[i], Field.Store.YES,
-              Field.Index.TOKENIZED));
-      modifier.addDocument(doc);
-    }
-    modifier.optimize();
+                         Field.Index.TOKENIZED));
+        modifier.addDocument(doc);
+      }
+      modifier.optimize();
 
-    Term term = new Term("city", "Amsterdam");
-    int hitCount = getHitCount(dir, term);
-    assertEquals(1, hitCount);
-    modifier.deleteDocuments(term);
-    hitCount = getHitCount(dir, term);
-    assertEquals(0, hitCount);
+      if (!autoCommit) {
+        modifier.close();
+      }
+
+      Term term = new Term("city", "Amsterdam");
+      int hitCount = getHitCount(dir, term);
+      assertEquals(1, hitCount);
+      if (!autoCommit) {
+        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
+        modifier.setUseCompoundFile(true);
+      }
+      modifier.deleteDocuments(term);
+      if (!autoCommit) {
+        modifier.close();
+      }
+      hitCount = getHitCount(dir, term);
+      assertEquals(0, hitCount);
 
-    modifier.close();
+      if (autoCommit) {
+        modifier.close();
+      }
+      dir.close();
+    }
   }
 
   // test when delete terms only apply to disk segments
   public void testNonRAMDelete() throws IOException {
-    Directory dir = new RAMDirectory();
-    IndexWriter modifier = new IndexWriter(dir,
-        new WhitespaceAnalyzer(), true);
-    modifier.setMaxBufferedDocs(2);
-    modifier.setMaxBufferedDeleteTerms(2);
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
 
-    int id = 0;
-    int value = 100;
+      Directory dir = new RAMDirectory();
+      IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                             new WhitespaceAnalyzer(), true);
+      modifier.setMaxBufferedDocs(2);
+      modifier.setMaxBufferedDeleteTerms(2);
 
-    for (int i = 0; i < 7; i++) {
-      addDoc(modifier, ++id, value);
-    }
-    modifier.flush();
+      int id = 0;
+      int value = 100;
+
+      for (int i = 0; i < 7; i++) {
+        addDoc(modifier, ++id, value);
+      }
+      modifier.flush();
 
-    assertEquals(0, modifier.getRamSegmentCount());
-    assertTrue(0 < modifier.getSegmentCount());
+      assertEquals(0, modifier.getRamSegmentCount());
+      assertTrue(0 < modifier.getSegmentCount());
 
-    IndexReader reader = IndexReader.open(dir);
-    assertEquals(7, reader.numDocs());
-    reader.close();
+      if (!autoCommit) {
+        modifier.close();
+      }
+
+      IndexReader reader = IndexReader.open(dir);
+      assertEquals(7, reader.numDocs());
+      reader.close();
+
+      if (!autoCommit) {
+        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
+        modifier.setMaxBufferedDocs(2);
+        modifier.setMaxBufferedDeleteTerms(2);
+      }
 
-    modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-    modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
 
-    reader = IndexReader.open(dir);
-    assertEquals(0, reader.numDocs());
-    reader.close();
+      if (!autoCommit) {
+        modifier.close();
+      }
 
-    modifier.close();
+      reader = IndexReader.open(dir);
+      assertEquals(0, reader.numDocs());
+      reader.close();
+      if (autoCommit) {
+        modifier.close();
+      }
+      dir.close();
+    }
   }
 
   // test when delete terms only apply to ram segments
   public void testRAMDeletes() throws IOException {
-    Directory dir = new RAMDirectory();
-    IndexWriter modifier = new IndexWriter(dir,
-        new WhitespaceAnalyzer(), true);
-    modifier.setMaxBufferedDocs(4);
-    modifier.setMaxBufferedDeleteTerms(4);
-
-    int id = 0;
-    int value = 100;
-
-    addDoc(modifier, ++id, value);
-    modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-    addDoc(modifier, ++id, value);
-    modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-
-    assertEquals(2, modifier.getNumBufferedDeleteTerms());
-    assertEquals(1, modifier.getBufferedDeleteTermsSize());
-
-    addDoc(modifier, ++id, value);
-    assertEquals(0, modifier.getSegmentCount());
-    modifier.flush();
-
-    IndexReader reader = IndexReader.open(dir);
-    assertEquals(1, reader.numDocs());
-
-    int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
-    assertEquals(1, hitCount);
-    reader.close();
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
+      Directory dir = new RAMDirectory();
+      IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                             new WhitespaceAnalyzer(), true);
+      modifier.setMaxBufferedDocs(4);
+      modifier.setMaxBufferedDeleteTerms(4);
+
+      int id = 0;
+      int value = 100;
+
+      addDoc(modifier, ++id, value);
+      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+      addDoc(modifier, ++id, value);
+      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+
+      assertEquals(2, modifier.getNumBufferedDeleteTerms());
+      assertEquals(1, modifier.getBufferedDeleteTermsSize());
+
+      addDoc(modifier, ++id, value);
+      assertEquals(0, modifier.getSegmentCount());
+      modifier.flush();
 
-    modifier.close();
+      if (!autoCommit) {
+        modifier.close();
+      }
+
+      IndexReader reader = IndexReader.open(dir);
+      assertEquals(1, reader.numDocs());
+
+      int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
+      assertEquals(1, hitCount);
+      reader.close();
+      if (autoCommit) {
+        modifier.close();
+      }
+      dir.close();
+    }
   }
 
   // test when delete terms apply to both disk and ram segments
   public void testBothDeletes() throws IOException {
-    Directory dir = new RAMDirectory();
-    IndexWriter modifier = new IndexWriter(dir,
-        new WhitespaceAnalyzer(), true);
-    modifier.setMaxBufferedDocs(100);
-    modifier.setMaxBufferedDeleteTerms(100);
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
 
-    int id = 0;
-    int value = 100;
+      Directory dir = new RAMDirectory();
+      IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                             new WhitespaceAnalyzer(), true);
+      modifier.setMaxBufferedDocs(100);
+      modifier.setMaxBufferedDeleteTerms(100);
 
-    for (int i = 0; i < 5; i++) {
-      addDoc(modifier, ++id, value);
-    }
+      int id = 0;
+      int value = 100;
 
-    value = 200;
-    for (int i = 0; i < 5; i++) {
-      addDoc(modifier, ++id, value);
-    }
-    modifier.flush();
+      for (int i = 0; i < 5; i++) {
+        addDoc(modifier, ++id, value);
+      }
 
-    for (int i = 0; i < 5; i++) {
-      addDoc(modifier, ++id, value);
-    }
-    modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-    modifier.flush();
+      value = 200;
+      for (int i = 0; i < 5; i++) {
+        addDoc(modifier, ++id, value);
+      }
+      modifier.flush();
 
-    IndexReader reader = IndexReader.open(dir);
-    assertEquals(5, reader.numDocs());
+      for (int i = 0; i < 5; i++) {
+        addDoc(modifier, ++id, value);
+      }
+      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
+
+      modifier.flush();
+      if (!autoCommit) {
+        modifier.close();
+      }
 
-    modifier.close();
+      IndexReader reader = IndexReader.open(dir);
+      assertEquals(5, reader.numDocs());
+      if (autoCommit) {
+        modifier.close();
+      }
+    }
   }
 
   // test that batched delete terms are flushed together
   public void testBatchDeletes() throws IOException {
-    Directory dir = new RAMDirectory();
-    IndexWriter modifier = new IndexWriter(dir,
-        new WhitespaceAnalyzer(), true);
-    modifier.setMaxBufferedDocs(2);
-    modifier.setMaxBufferedDeleteTerms(2);
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
+      Directory dir = new RAMDirectory();
+      IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                             new WhitespaceAnalyzer(), true);
+      modifier.setMaxBufferedDocs(2);
+      modifier.setMaxBufferedDeleteTerms(2);
 
-    int id = 0;
-    int value = 100;
+      int id = 0;
+      int value = 100;
 
-    for (int i = 0; i < 7; i++) {
-      addDoc(modifier, ++id, value);
-    }
-    modifier.flush();
+      for (int i = 0; i < 7; i++) {
+        addDoc(modifier, ++id, value);
+      }
+      modifier.flush();
+      if (!autoCommit) {
+        modifier.close();
+      }
 
-    IndexReader reader = IndexReader.open(dir);
-    assertEquals(7, reader.numDocs());
-    reader.close();
-
-    id = 0;
-    modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
-    modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
-
-    reader = IndexReader.open(dir);
-    assertEquals(5, reader.numDocs());
-    reader.close();
-
-    Term[] terms = new Term[3];
-    for (int i = 0; i < terms.length; i++) {
-      terms[i] = new Term("id", String.valueOf(++id));
-    }
-    modifier.deleteDocuments(terms);
+      IndexReader reader = IndexReader.open(dir);
+      assertEquals(7, reader.numDocs());
+      reader.close();
+      
+      if (!autoCommit) {
+        modifier = new IndexWriter(dir, autoCommit,
+                                   new WhitespaceAnalyzer());
+        modifier.setMaxBufferedDocs(2);
+        modifier.setMaxBufferedDeleteTerms(2);
+      }
+
+      id = 0;
+      modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
+      modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
 
-    reader = IndexReader.open(dir);
-    assertEquals(2, reader.numDocs());
-    reader.close();
+      if (!autoCommit) {
+        modifier.close();
+      }
 
-    modifier.close();
+      reader = IndexReader.open(dir);
+      assertEquals(5, reader.numDocs());
+      reader.close();
+
+      Term[] terms = new Term[3];
+      for (int i = 0; i < terms.length; i++) {
+        terms[i] = new Term("id", String.valueOf(++id));
+      }
+      if (!autoCommit) {
+        modifier = new IndexWriter(dir, autoCommit,
+                                   new WhitespaceAnalyzer());
+        modifier.setMaxBufferedDocs(2);
+        modifier.setMaxBufferedDeleteTerms(2);
+      }
+      modifier.deleteDocuments(terms);
+      if (!autoCommit) {
+        modifier.close();
+      }
+      reader = IndexReader.open(dir);
+      assertEquals(2, reader.numDocs());
+      reader.close();
+
+      if (autoCommit) {
+        modifier.close();
+      }
+      dir.close();
+    }
   }
 
   private void addDoc(IndexWriter modifier, int id, int value)
@@ -233,201 +317,203 @@
     int START_COUNT = 157;
     int END_COUNT = 144;
 
-    // First build up a starting index:
-    RAMDirectory startDir = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(startDir, new WhitespaceAnalyzer(),
-        true);
-    for (int i = 0; i < 157; i++) {
-      Document d = new Document();
-      d.add(new Field("id", Integer.toString(i), Field.Store.YES,
-          Field.Index.UN_TOKENIZED));
-      d.add(new Field("content", "aaa " + i, Field.Store.NO,
-          Field.Index.TOKENIZED));
-      writer.addDocument(d);
-    }
-    writer.close();
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = (0==pass);
 
-    long diskUsage = startDir.sizeInBytes();
-    long diskFree = diskUsage + 10;
+      // First build up a starting index:
+      RAMDirectory startDir = new RAMDirectory();
+      IndexWriter writer = new IndexWriter(startDir, autoCommit,
+                                           new WhitespaceAnalyzer(), true);
+      for (int i = 0; i < 157; i++) {
+        Document d = new Document();
+        d.add(new Field("id", Integer.toString(i), Field.Store.YES,
+                        Field.Index.UN_TOKENIZED));
+        d.add(new Field("content", "aaa " + i, Field.Store.NO,
+                        Field.Index.TOKENIZED));
+        writer.addDocument(d);
+      }
+      writer.close();
 
-    IOException err = null;
+      long diskUsage = startDir.sizeInBytes();
+      long diskFree = diskUsage + 10;
 
-    boolean done = false;
+      IOException err = null;
 
-    // Iterate w/ ever increasing free disk space:
-    while (!done) {
-      MockRAMDirectory dir = new MockRAMDirectory(startDir);
-      IndexWriter modifier = new IndexWriter(dir,
-          new WhitespaceAnalyzer(), false);
+      boolean done = false;
 
-      modifier.setMaxBufferedDocs(1000); // use flush or close
-      modifier.setMaxBufferedDeleteTerms(1000); // use flush or close
+      // Iterate w/ ever increasing free disk space:
+      while (!done) {
+        MockRAMDirectory dir = new MockRAMDirectory(startDir);
+        IndexWriter modifier = new IndexWriter(dir, autoCommit,
+                                               new WhitespaceAnalyzer());
 
-      // For each disk size, first try to commit against
-      // dir that will hit random IOExceptions & disk
-      // full; after, give it infinite disk space & turn
-      // off random IOExceptions & retry w/ same reader:
-      boolean success = false;
+        modifier.setMaxBufferedDocs(1000); // use flush or close
+        modifier.setMaxBufferedDeleteTerms(1000); // use flush or close
 
-      for (int x = 0; x < 2; x++) {
+        // For each disk size, first try to commit against
+        // dir that will hit random IOExceptions & disk
+        // full; after, give it infinite disk space & turn
+        // off random IOExceptions & retry w/ same reader:
+        boolean success = false;
 
-        double rate = 0.1;
-        double diskRatio = ((double)diskFree) / diskUsage;
-        long thisDiskFree;
-        String testName;
+        for (int x = 0; x < 2; x++) {
 
-        if (0 == x) {
-          thisDiskFree = diskFree;
-          if (diskRatio >= 2.0) {
-            rate /= 2;
-          }
-          if (diskRatio >= 4.0) {
-            rate /= 2;
-          }
-          if (diskRatio >= 6.0) {
-            rate = 0.0;
-          }
-          if (debug) {
-            System.out.println("\ncycle: " + diskFree + " bytes");
-          }
-          testName = "disk full during reader.close() @ " + thisDiskFree
+          double rate = 0.1;
+          double diskRatio = ((double)diskFree) / diskUsage;
+          long thisDiskFree;
+          String testName;
+
+          if (0 == x) {
+            thisDiskFree = diskFree;
+            if (diskRatio >= 2.0) {
+              rate /= 2;
+            }
+            if (diskRatio >= 4.0) {
+              rate /= 2;
+            }
+            if (diskRatio >= 6.0) {
+              rate = 0.0;
+            }
+            if (debug) {
+              System.out.println("\ncycle: " + diskFree + " bytes");
+            }
+            testName = "disk full during reader.close() @ " + thisDiskFree
               + " bytes";
-        } else {
-          thisDiskFree = 0;
-          rate = 0.0;
-          if (debug) {
-            System.out.println("\ncycle: same writer: unlimited disk space");
+          } else {
+            thisDiskFree = 0;
+            rate = 0.0;
+            if (debug) {
+              System.out.println("\ncycle: same writer: unlimited disk space");
+            }
+            testName = "reader re-use after disk full";
           }
-          testName = "reader re-use after disk full";
-        }
 
-        dir.setMaxSizeInBytes(thisDiskFree);
-        dir.setRandomIOExceptionRate(rate, diskFree);
+          dir.setMaxSizeInBytes(thisDiskFree);
+          dir.setRandomIOExceptionRate(rate, diskFree);
 
-        try {
-          if (0 == x) {
-            int docId = 12;
-            for (int i = 0; i < 13; i++) {
-              if (updates) {
-                Document d = new Document();
-                d.add(new Field("id", Integer.toString(i), Field.Store.YES,
-                    Field.Index.UN_TOKENIZED));
-                d.add(new Field("content", "bbb " + i, Field.Store.NO,
-                    Field.Index.TOKENIZED));
-                modifier.updateDocument(new Term("id", Integer.toString(docId)), d);
-              } else { // deletes
-                modifier.deleteDocuments(new Term("id", Integer.toString(docId)));
-                // modifier.setNorm(docId, "contents", (float)2.0);
+          try {
+            if (0 == x) {
+              int docId = 12;
+              for (int i = 0; i < 13; i++) {
+                if (updates) {
+                  Document d = new Document();
+                  d.add(new Field("id", Integer.toString(i), Field.Store.YES,
+                                  Field.Index.UN_TOKENIZED));
+                  d.add(new Field("content", "bbb " + i, Field.Store.NO,
+                                  Field.Index.TOKENIZED));
+                  modifier.updateDocument(new Term("id", Integer.toString(docId)), d);
+                } else { // deletes
+                  modifier.deleteDocuments(new Term("id", Integer.toString(docId)));
+                  // modifier.setNorm(docId, "contents", (float)2.0);
+                }
+                docId += 12;
               }
-              docId += 12;
+            }
+            modifier.close();
+            success = true;
+            if (0 == x) {
+              done = true;
             }
           }
-          modifier.close();
-          success = true;
-          if (0 == x) {
-            done = true;
+          catch (IOException e) {
+            if (debug) {
+              System.out.println("  hit IOException: " + e);
+            }
+            err = e;
+            if (1 == x) {
+              e.printStackTrace();
+              fail(testName + " hit IOException after disk space was freed up");
+            }
           }
-        }
-        catch (IOException e) {
-          if (debug) {
-            System.out.println("  hit IOException: " + e);
+
+          // Whether we succeeded or failed, check that all
+          // un-referenced files were in fact deleted (ie,
+          // we did not create garbage). Just create a
+          // new IndexFileDeleter, have it delete
+          // unreferenced files, then verify that in fact
+          // no files were deleted:
+          String[] startFiles = dir.list();
+          SegmentInfos infos = new SegmentInfos();
+          infos.read(dir);
+          IndexFileDeleter d = new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
+          String[] endFiles = dir.list();
+
+          Arrays.sort(startFiles);
+          Arrays.sort(endFiles);
+
+          // for(int i=0;i<startFiles.length;i++) {
+          // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
+          // }
+
+          if (!Arrays.equals(startFiles, endFiles)) {
+            String successStr;
+            if (success) {
+              successStr = "success";
+            } else {
+              successStr = "IOException";
+              err.printStackTrace();
+            }
+            fail("reader.close() failed to delete unreferenced files after "
+                 + successStr + " (" + diskFree + " bytes): before delete:\n    "
+                 + arrayToString(startFiles) + "\n  after delete:\n    "
+                 + arrayToString(endFiles));
+          }
+
+          // Finally, verify index is not corrupt, and, if
+          // we succeeded, we see all docs changed, and if
+          // we failed, we see either all docs or no docs
+          // changed (transactional semantics):
+          IndexReader newReader = null;
+          try {
+            newReader = IndexReader.open(dir);
           }
-          err = e;
-          if (1 == x) {
+          catch (IOException e) {
             e.printStackTrace();
-            fail(testName + " hit IOException after disk space was freed up");
+            fail(testName
+                 + ":exception when creating IndexReader after disk full during close: "
+                 + e);
           }
-        }
 
-        // Whether we succeeded or failed, check that all
-        // un-referenced files were in fact deleted (ie,
-        // we did not create garbage). Just create a
-        // new IndexFileDeleter, have it delete
-        // unreferenced files, then verify that in fact
-        // no files were deleted:
-        String[] startFiles = dir.list();
-        SegmentInfos infos = new SegmentInfos();
-        infos.read(dir);
-        IndexFileDeleter d = new IndexFileDeleter(infos, dir);
-        d.findDeletableFiles();
-        d.deleteFiles();
-        String[] endFiles = dir.list();
-
-        Arrays.sort(startFiles);
-        Arrays.sort(endFiles);
-
-        // for(int i=0;i<startFiles.length;i++) {
-        // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
-        // }
-
-        if (!Arrays.equals(startFiles, endFiles)) {
-          String successStr;
+          IndexSearcher searcher = new IndexSearcher(newReader);
+          Hits hits = null;
+          try {
+            hits = searcher.search(new TermQuery(searchTerm));
+          }
+          catch (IOException e) {
+            e.printStackTrace();
+            fail(testName + ": exception when searching: " + e);
+          }
+          int result2 = hits.length();
           if (success) {
-            successStr = "success";
+            if (result2 != END_COUNT) {
+              fail(testName
+                   + ": method did not throw exception but hits.length for search on term 'aaa' is "
+                   + result2 + " instead of expected " + END_COUNT);
+            }
           } else {
-            successStr = "IOException";
-            err.printStackTrace();
+            // On hitting exception we still may have added
+            // all docs:
+            if (result2 != START_COUNT && result2 != END_COUNT) {
+              err.printStackTrace();
+              fail(testName
+                   + ": method did throw exception but hits.length for search on term 'aaa' is "
+                   + result2 + " instead of expected " + START_COUNT);
+            }
           }
-          fail("reader.close() failed to delete unreferenced files after "
-              + successStr + " (" + diskFree + " bytes): before delete:\n    "
-              + arrayToString(startFiles) + "\n  after delete:\n    "
-              + arrayToString(endFiles));
-        }
 
-        // Finally, verify index is not corrupt, and, if
-        // we succeeded, we see all docs changed, and if
-        // we failed, we see either all docs or no docs
-        // changed (transactional semantics):
-        IndexReader newReader = null;
-        try {
-          newReader = IndexReader.open(dir);
-        }
-        catch (IOException e) {
-          e.printStackTrace();
-          fail(testName
-              + ":exception when creating IndexReader after disk full during close: "
-              + e);
-        }
+          searcher.close();
+          newReader.close();
 
-        IndexSearcher searcher = new IndexSearcher(newReader);
-        Hits hits = null;
-        try {
-          hits = searcher.search(new TermQuery(searchTerm));
-        }
-        catch (IOException e) {
-          e.printStackTrace();
-          fail(testName + ": exception when searching: " + e);
-        }
-        int result2 = hits.length();
-        if (success) {
-          if (result2 != END_COUNT) {
-            fail(testName
-                + ": method did not throw exception but hits.length for search on term 'aaa' is "
-                + result2 + " instead of expected " + END_COUNT);
-          }
-        } else {
-          // On hitting exception we still may have added
-          // all docs:
-          if (result2 != START_COUNT && result2 != END_COUNT) {
-            err.printStackTrace();
-            fail(testName
-                + ": method did throw exception but hits.length for search on term 'aaa' is "
-                + result2 + " instead of expected " + START_COUNT);
+          if (result2 == END_COUNT) {
+            break;
           }
         }
 
-        searcher.close();
-        newReader.close();
+        dir.close();
 
-        if (result2 == END_COUNT) {
-          break;
-        }
+        // Try again with 10 more bytes of free space:
+        diskFree += 10;
       }
-
-      dir.close();
-
-      // Try again with 10 more bytes of free space:
-      diskFree += 10;
     }
   }
 

Modified: lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java?view=diff&rev=517599&r1=517598&r2=517599
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java Tue Mar 13 02:06:22 2007
@@ -68,7 +68,7 @@
       if (realUsage > dir.maxUsedSize) {
         dir.maxUsedSize = realUsage;
       }
-      throw new IOException("fake disk full at " + dir.sizeInBytes() + " bytes");
+      throw new IOException("fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes");
     } else {
       super.flushBuffer(src, len);
     }



Mime
View raw message