lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r909357 - in /lucene/java/trunk: ./ src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Date Fri, 12 Feb 2010 11:07:32 GMT
Author: mikemccand
Date: Fri Feb 12 11:07:31 2010
New Revision: 909357

URL: http://svn.apache.org/viewvc?rev=909357&view=rev
Log:
LUCENE-2209: add expert IndexWriter.removeUnusedFiles to force IW to retry deletion of files
no longer used (only useful on windows)

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/trunk/src/java/org/apache/lucene/index/IndexFileDeleter.java
    lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=909357&r1=909356&r2=909357&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Fri Feb 12 11:07:31 2010
@@ -64,6 +64,13 @@
 * LUCENE-2240: SimpleAnalyzer and WhitespaceAnalyzer now have
   Version ctors.  (Simon Willnauer via Uwe Schindler)
 
+* LUCENE-2259: Add IndexWriter.removeUnusedFiles, to attempt removing
+  unused files.  This is only useful on Windows, which prevents
+  deletion of open files. IndexWriter will eventually remove these
+  files itself; this method just lets you do so when you know the
+  files are no longer open by IndexReaders. (luocanrao via Mike
+  McCandless)
+
 Bug fixes
 
 * LUCENE-2092: BooleanQuery was ignoring disableCoord in its hashCode

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java?rev=909357&r1=909356&r2=909357&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/DirectoryReader.java Fri Feb 12 11:07:31
2010
@@ -883,6 +883,12 @@
     // not a good idea):
     FieldCache.DEFAULT.purge(this);
 
+    if (writer != null) {
+      // Since we just closed, writer may now be able to
+      // delete unused files:
+      writer.deleteUnusedFiles();
+    }
+
     // throw the first exception
     if (ioe != null) throw ioe;
   }

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=909357&r1=909356&r2=909357&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/IndexFileDeleter.java Fri Feb 12 11:07:31
2010
@@ -338,7 +338,7 @@
     deletePendingFiles();
   }
 
-  private void deletePendingFiles() throws IOException {
+  public void deletePendingFiles() throws IOException {
     if (deletable != null) {
       List<String> oldDeletable = deletable;
       deletable = null;

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java?rev=909357&r1=909356&r2=909357&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java Fri Feb 12 11:07:31
2010
@@ -4883,4 +4883,25 @@
   synchronized boolean isClosed() {
     return closed;
   }
+
+  /** Expert: remove any index files that are no longer
+   *  used.
+   * 
+   *  <p> IndexWriter normally deletes unused files itself,
+   *  during indexing.  However, on Windows, which disallows
+   *  deletion of open files, if there is a reader open on
+   *  the index then those files cannot be deleted.  This is
+   *  fine, because IndexWriter will periodically retry
+   *  the deletion.</p>
+   *
+   *  <p> However, IndexWriter doesn't try that often: only
+   *  on open, close, flushing a new segment, and finishing
+   *  a merge.  If you don't do any of these actions with your
+   *  IndexWriter, you'll see the unused files linger.  If
+   *  that's a problem, call this method to delete them
+   *  (once you've closed the open readers that were
+   *  preventing their deletion). */
+  public synchronized void deleteUnusedFiles() throws IOException {
+    deleter.deletePendingFiles();
+  }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=909357&r1=909356&r2=909357&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Feb 12 11:07:31
2010
@@ -4670,4 +4670,62 @@
     dir.close();
     assertFalse(failed.get());
   }
+
+  public void testDeleteUnusedFiles() throws Exception {
+
+    for(int iter=0;iter<2;iter++) {
+      Directory dir = new MockRAMDirectory();
+      IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
+      Document doc = new Document();
+      doc.add(new Field("field", "go", Field.Store.NO, Field.Index.ANALYZED));
+      w.addDocument(doc);
+      IndexReader r;
+      if (iter == 0) {
+        // use NRT
+        r = w.getReader();
+      } else {
+        // don't use NRT
+        w.commit();
+        r = IndexReader.open(dir);
+      }
+
+      List<String> files = Arrays.asList(dir.listAll());
+      assertTrue(files.contains("_0.cfs"));
+      w.addDocument(doc);
+      w.optimize();
+      if (iter == 1) {
+        w.commit();
+      }
+      IndexReader r2 = r.reopen();
+      assertTrue(r != r2);
+      files = Arrays.asList(dir.listAll());
+      assertTrue(files.contains("_0.cfs"));
+      // optimize created this
+      assertTrue(files.contains("_2.cfs"));
+      w.deleteUnusedFiles();
+
+      files = Arrays.asList(dir.listAll());
+      // r still holds this file open
+      assertTrue(files.contains("_0.cfs"));
+      assertTrue(files.contains("_2.cfs"));
+
+      r.close();
+      if (iter == 0) {
+        // on closing NRT reader, it calls writer.deleteUnusedFiles
+        files = Arrays.asList(dir.listAll());
+        assertFalse(files.contains("_0.cfs"));
+      } else {
+        // now writer can remove it
+        w.deleteUnusedFiles();
+        files = Arrays.asList(dir.listAll());
+        assertFalse(files.contains("_0.cfs"));
+      }
+      assertTrue(files.contains("_2.cfs"));
+
+      w.close();
+      r2.close();
+
+      dir.close();
+    }
+  }
 }



Mime
View raw message