lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r984217 - in /lucene/java/branches/lucene_3_0: ./ contrib/ contrib/analyzers/ contrib/analyzers/common/ contrib/analyzers/common/src/test/ contrib/analyzers/smartcn/ contrib/highlighter/ contrib/highlighter/src/test/ contrib/instantiated/sr...
Date Tue, 10 Aug 2010 21:24:03 GMT
Author: mikemccand
Date: Tue Aug 10 21:24:02 2010
New Revision: 984217

URL: http://svn.apache.org/viewvc?rev=984217&view=rev
Log:
LUCENE-2593 (on 30x): fix certain rare cases where disk full could cause index corruption

Modified:
    lucene/java/branches/lucene_3_0/   (props changed)
    lucene/java/branches/lucene_3_0/CHANGES.txt   (contents, props changed)
    lucene/java/branches/lucene_3_0/build.xml   (props changed)
    lucene/java/branches/lucene_3_0/contrib/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/CHANGES.txt   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/common/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/common/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/smartcn/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/highlighter/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/   (props changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java   (props
changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexFileDeleter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
  (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
  (props changed)

Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene:941394,946651,948430,957490
-/lucene/dev/trunk/lucene:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4:748824
 /lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336

Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Tue Aug 10 21:24:02 2010
@@ -27,6 +27,9 @@ Bug fixes
   and float were not affected.
   (Yonik Seeley, Uwe Schindler)
 
+* LUCENE-2593: Fixed certain rare cases where a disk full could lead
+  to a corrupted index (Robert Muir, Mike McCandless)
+
 ======================= Release 3.0.2 2010-06-18 =======================
 
 Changes in backwards compatibility policy

Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/CHANGES.txt:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/CHANGES.txt:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/build.xml:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/build.xml:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/build.xml:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/build.xml:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/build.xml:748824
 /lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/contrib:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/contrib:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/contrib:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/contrib:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/contrib:748824
 /lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/contrib/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/contrib/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
 /lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/analyzers:941394,946651,957490
-/lucene/dev/trunk/lucene/contrib/analyzers:929738,935522,939611,940730,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/contrib/analyzers:941394,946651,957490,984210
+/lucene/dev/trunk/lucene/contrib/analyzers:929738,935522,939611,940730,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/trunk/contrib/analyzers:887670,889579,889866,890439,891205,891377,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/common/
------------------------------------------------------------------------------
    svn:mergeinfo = /lucene/dev/trunk/lucene/contrib/analyzers/common:984187

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/common/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -0,0 +1 @@
+/lucene/dev/trunk/lucene/contrib/analyzers/common/src/test:984187

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/smartcn/
------------------------------------------------------------------------------
    svn:mergeinfo = /lucene/dev/trunk/lucene/contrib/analyzers/smartcn:984187

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/
------------------------------------------------------------------------------
    svn:mergeinfo = /lucene/dev/trunk/lucene/contrib/highlighter:984187

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/contrib/highlighter/src/test:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/contrib/highlighter/src/test:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
 /lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336

Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103
+/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187
 /lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
 /lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src:748824
 /lucene/java/branches/lucene_2_9/src:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DirectoryReader.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DirectoryReader.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/DirectoryReader.java
Tue Aug 10 21:24:02 2010
@@ -55,7 +55,6 @@ class DirectoryReader extends IndexReade
   private final int termInfosIndexDivisor;
 
   private boolean rollbackHasChanges;
-  private SegmentInfos rollbackSegmentInfos;
 
   private SegmentReader[] subReaders;
   private int[] starts;                           // 1st docno for each segment
@@ -780,7 +779,6 @@ class DirectoryReader extends IndexReade
 
   void startCommit() {
     rollbackHasChanges = hasChanges;
-    rollbackSegmentInfos = (SegmentInfos) segmentInfos.clone();
     for (int i = 0; i < subReaders.length; i++) {
       subReaders[i].startCommit();
     }
@@ -788,14 +786,6 @@ class DirectoryReader extends IndexReade
 
   void rollbackCommit() {
     hasChanges = rollbackHasChanges;
-    for (int i = 0; i < segmentInfos.size(); i++) {
-      // Rollback each segmentInfo.  Because the
-      // SegmentReader holds a reference to the
-      // SegmentInfo we can't [easily] just replace
-      // segmentInfos, so we reset it in place instead:
-      segmentInfos.info(i).reset(rollbackSegmentInfos.info(i));
-    }
-    rollbackSegmentInfos = null;
     for (int i = 0; i < subReaders.length; i++) {
       subReaders[i].rollbackCommit();
     }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexFileDeleter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexFileDeleter.java
Tue Aug 10 21:24:02 2010
@@ -469,6 +469,14 @@ final class IndexFileDeleter {
     }
   }
 
+  public boolean exists(String fileName) {
+    if (!refCounts.containsKey(fileName)) {
+      return false;
+    } else {
+      return getRefCount(fileName).count > 0;
+    }
+  }
+
   private RefCount getRefCount(String fileName) {
     RefCount rc;
     if (!refCounts.containsKey(fileName)) {

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/IndexWriter.java Tue
Aug 10 21:24:02 2010
@@ -474,7 +474,7 @@ public class IndexWriter implements Clos
 
       final boolean pooled = readerMap.containsKey(sr.getSegmentInfo());
 
-      assert !pooled | readerMap.get(sr.getSegmentInfo()) == sr;
+      assert !pooled || readerMap.get(sr.getSegmentInfo()) == sr;
 
       // Drop caller's ref; for an external reader (not
       // pooled), this decRef will close it
@@ -482,28 +482,30 @@ public class IndexWriter implements Clos
 
       if (pooled && (drop || (!poolReaders && sr.getRefCount() == 1))) {
 
-        // We are the last ref to this reader; since we're
-        // not pooling readers, we release it:
-        readerMap.remove(sr.getSegmentInfo());
-
+        // We invoke deleter.checkpoint below, so we must be
+        // sync'd on IW if there are changes:
         assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this);
 
+        // Discard (don't save) changes when we are dropping
+        // the reader; this is used only on the sub-readers
+        // after a successful merge.
+        sr.hasChanges &= !drop;
+
+        final boolean hasChanges = sr.hasChanges;
+
         // Drop our ref -- this will commit any pending
         // changes to the dir
-        boolean success = false;
-        try {
-          sr.close();
-          success = true;
-        } finally {
-          if (!success && sr.hasChanges) {
-            // Abandon the changes & retry closing:
-            sr.hasChanges = false;
-            try {
-              sr.close();
-            } catch (Throwable ignore) {
-              // Keep throwing original exception
-            }
-          }
+        sr.close();
+
+        // We are the last ref to this reader; since we're
+        // not pooling readers, we release it:
+        readerMap.remove(sr.getSegmentInfo());
+
+        if (hasChanges) {
+          // Must checkpoint w/ deleter, because this
+          // segment reader will have created new _X_N.del
+          // file.
+          deleter.checkpoint(segmentInfos, false);
         }
       }
     }
@@ -511,6 +513,10 @@ public class IndexWriter implements Clos
     /** Remove all our references to readers, and commits
      *  any pending changes. */
     synchronized void close() throws IOException {
+      // We invoke deleter.checkpoint below, so we must be
+      // sync'd on IW:
+      assert Thread.holdsLock(IndexWriter.this);
+
       Iterator<Map.Entry<SegmentInfo,SegmentReader>> iter = readerMap.entrySet().iterator();
       while (iter.hasNext()) {
         
@@ -519,16 +525,12 @@ public class IndexWriter implements Clos
         SegmentReader sr = ent.getValue();
         if (sr.hasChanges) {
           assert infoIsLive(sr.getSegmentInfo());
-          sr.startCommit();
-          boolean success = false;
-          try {
-            sr.doCommit(null);
-            success = true;
-          } finally {
-            if (!success) {
-              sr.rollbackCommit();
-            }
-          }
+          sr.doCommit(null);
+
+          // Must checkpoint w/ deleter, because this
+          // segment reader will have created new _X_N.del
+          // file.
+          deleter.checkpoint(segmentInfos, false);
         }
 
         iter.remove();
@@ -546,21 +548,22 @@ public class IndexWriter implements Clos
      * @throws IOException
      */
     synchronized void commit() throws IOException {
+
+      // We invoke deleter.checkpoint below, so we must be
+      // sync'd on IW:
+      assert Thread.holdsLock(IndexWriter.this);
+
       for (Map.Entry<SegmentInfo,SegmentReader> ent : readerMap.entrySet()) {
 
         SegmentReader sr = ent.getValue();
         if (sr.hasChanges) {
           assert infoIsLive(sr.getSegmentInfo());
-          sr.startCommit();
-          boolean success = false;
-          try {
-            sr.doCommit(null);
-            success = true;
-          } finally {
-            if (!success) {
-              sr.rollbackCommit();
-            }
-          }
+          sr.doCommit(null);
+
+          // Must checkpoint w/ deleter, because this
+          // segment reader will have created new _X_N.del
+          // file.
+          deleter.checkpoint(segmentInfos, false);
         }
       }
     }
@@ -4412,7 +4415,7 @@ public class IndexWriter implements Clos
           for (int i=0;i<numSegments;i++) {
             if (merge.readers[i] != null) {
               try {
-                readerPool.release(merge.readers[i], true);
+                readerPool.release(merge.readers[i], false);
               } catch (Throwable t) {
               }
             }
@@ -4513,32 +4516,14 @@ public class IndexWriter implements Clos
   private final synchronized boolean applyDeletes() throws CorruptIndexException, IOException
{
     assert testPoint("startApplyDeletes");
     flushDeletesCount++;
-    SegmentInfos rollback = (SegmentInfos) segmentInfos.clone();
     boolean success = false;
     boolean changed;
     try {
       changed = docWriter.applyDeletes(segmentInfos);
       success = true;
     } finally {
-      if (!success) {
-        if (infoStream != null)
-          message("hit exception flushing deletes");
-
-        // Carefully remove any partially written .del
-        // files
-        final int size = rollback.size();
-        for(int i=0;i<size;i++) {
-          final String newDelFileName = segmentInfos.info(i).getDelFileName();
-          final String delFileName = rollback.info(i).getDelFileName();
-          if (newDelFileName != null && !newDelFileName.equals(delFileName))
-            deleter.deleteFile(newDelFileName);
-        }
-
-        // Fully replace the segmentInfos since flushed
-        // deletes could have changed any of the
-        // SegmentInfo instances:
-        segmentInfos.clear();
-        segmentInfos.addAll(rollback);
+      if (!success && infoStream != null) {
+        message("hit exception flushing deletes");
       }
     }
 
@@ -4694,30 +4679,37 @@ public class IndexWriter implements Clos
             return;
           }
 
+
           // First, we clone & incref the segmentInfos we intend
           // to sync, then, without locking, we sync() each file
           // referenced by toSync, in the background.  Multiple
           // threads can be doing this at once, if say a large
           // merge and a small merge finish at the same time:
-
+        
           if (infoStream != null)
             message("startCommit index=" + segString(segmentInfos) + " changeCount=" + changeCount);
-          
+        
           readerPool.commit();
-          
+        
           toSync = (SegmentInfos) segmentInfos.clone();
-
+        
           if (commitUserData != null)
             toSync.setUserData(commitUserData);
-
+        
           deleter.incRef(toSync, false);
           myChangeCount = changeCount;
-
+        
           Collection<String> files = toSync.files(directory, false);
           for(final String fileName: files) {
             assert directory.fileExists(fileName): "file " + fileName + " does not exist";
-          }
 
+            // If this trips it means we are missing a call to
+            // .checkpoint somewhere, because by the time we
+            // are called, deleter should know about every
+            // file referenced by the current head
+            // segmentInfos:
+            assert deleter.exists(fileName);
+          }
         } finally {
           resumeAddIndexes();
         }

Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/index/SegmentReader.java Tue
Aug 10 21:24:02 2010
@@ -62,6 +62,7 @@ public class SegmentReader extends Index
   private boolean rollbackHasChanges = false;
   private boolean rollbackDeletedDocsDirty = false;
   private boolean rollbackNormsDirty = false;
+  private SegmentInfo rollbackSegmentInfo;
   private int rollbackPendingDeleteCount;
 
   // optionally used for the .nrm file shared by multiple norms
@@ -548,11 +549,25 @@ public class SegmentReader extends Index
 
       // NOTE: norms are re-written in regular directory, not cfs
       si.advanceNormGen(this.number);
-      IndexOutput out = directory().createOutput(si.getNormFileName(this.number));
+      final String normFileName = si.getNormFileName(this.number);
+      IndexOutput out = directory().createOutput(normFileName);
+      boolean success = false;
       try {
-        out.writeBytes(bytes, maxDoc());
+        try {
+          out.writeBytes(bytes, maxDoc());
+        } finally {
+          out.close();
+        }
+        success = true;
       } finally {
-        out.close();
+        if (!success) {
+          try {
+            directory().deleteFile(normFileName);
+          } catch (Throwable t) {
+            // suppress this so we keep throwing the
+            // original exception
+          }
+        }
       }
       this.dirty = false;
     }
@@ -767,33 +782,60 @@ public class SegmentReader extends Index
   @Override
   protected void doCommit(Map<String,String> commitUserData) throws IOException {
     if (hasChanges) {
-      if (deletedDocsDirty) {               // re-write deleted
-        si.advanceDelGen();
-
-        // We can write directly to the actual name (vs to a
-        // .tmp & renaming it) because the file is not live
-        // until segments file is written:
-        deletedDocs.write(directory(), si.getDelFileName());
-
-        si.setDelCount(si.getDelCount()+pendingDeleteCount);
-        pendingDeleteCount = 0;
-        assert deletedDocs.count() == si.getDelCount(): "delete count mismatch during commit:
info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
-      } else {
-        assert pendingDeleteCount == 0;
+      startCommit();
+      boolean success = false;
+      try {
+        commitChanges(commitUserData);
+        success = true;
+      } finally {
+        if (!success) {
+          rollbackCommit();
+        }
       }
+    }
+  }
+
+  private void commitChanges(Map<String,String> commitUserData) throws IOException
{
+    if (deletedDocsDirty) {               // re-write deleted
+      si.advanceDelGen();
 
-      if (normsDirty) {               // re-write norms
-        si.setNumFields(core.fieldInfos.size());
-        for (final Norm norm : norms.values()) {
-          if (norm.dirty) {
-            norm.reWrite(si);
+      // We can write directly to the actual name (vs to a
+      // .tmp & renaming it) because the file is not live
+      // until segments file is written:
+      final String delFileName = si.getDelFileName();
+      boolean success = false;
+      try {
+        deletedDocs.write(directory(), delFileName);
+        success = true;
+      } finally {
+        if (!success) {
+          try {
+            directory().deleteFile(delFileName);
+          } catch (Throwable t) {
+            // suppress this so we keep throwing the
+            // original exception
           }
         }
       }
-      deletedDocsDirty = false;
-      normsDirty = false;
-      hasChanges = false;
+
+      si.setDelCount(si.getDelCount()+pendingDeleteCount);
+      pendingDeleteCount = 0;
+      assert deletedDocs.count() == si.getDelCount(): "delete count mismatch during commit:
info=" + si.getDelCount() + " vs BitVector=" + deletedDocs.count();
+    } else {
+      assert pendingDeleteCount == 0;
+    }
+
+    if (normsDirty) {               // re-write norms
+      si.setNumFields(core.fieldInfos.size());
+      for (final Norm norm : norms.values()) {
+        if (norm.dirty) {
+          norm.reWrite(si);
+        }
+      }
     }
+    deletedDocsDirty = false;
+    normsDirty = false;
+    hasChanges = false;
   }
 
   FieldsReader getFieldsReader() {
@@ -1241,6 +1283,7 @@ public class SegmentReader extends Index
   }
 
   void startCommit() {
+    rollbackSegmentInfo = (SegmentInfo) si.clone();
     rollbackHasChanges = hasChanges;
     rollbackDeletedDocsDirty = deletedDocsDirty;
     rollbackNormsDirty = normsDirty;
@@ -1251,6 +1294,7 @@ public class SegmentReader extends Index
   }
 
   void rollbackCommit() {
+    si.reset(rollbackSegmentInfo);
     hasChanges = rollbackHasChanges;
     deletedDocsDirty = rollbackDeletedDocsDirty;
     normsDirty = rollbackNormsDirty;

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,3 +1,3 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index:941394,946651,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:929738,935522,940730,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index:941394,946651,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:929738,935522,940730,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/trunk/src/test/org/apache/lucene/index:887670,889579,889866,890439,891205,891377,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Modified: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriter.java
(original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriter.java
Tue Aug 10 21:24:02 2010
@@ -4672,4 +4672,66 @@ public class TestIndexWriter extends Luc
     dir.close();
     assertFalse(failed.get());
   }
+
+  private static class FailTwiceDuringMerge extends MockRAMDirectory.Failure {
+    public boolean didFail1;
+    public boolean didFail2;
+
+    @Override
+    public void eval(MockRAMDirectory dir)  throws IOException {
+      if (!doFail) {
+        return;
+      }
+      StackTraceElement[] trace = new Exception().getStackTrace();
+      for (int i = 0; i < trace.length; i++) {
+        if ("org.apache.lucene.index.SegmentMerger".equals(trace[i].getClassName()) &&
"mergeTerms".equals(trace[i].getMethodName()) && !didFail1) {
+          didFail1 = true;
+          throw new IOException("fake disk full during mergeTerms");
+        }
+        if ("org.apache.lucene.util.BitVector".equals(trace[i].getClassName()) &&
"write".equals(trace[i].getMethodName()) && !didFail2) {
+          didFail2 = true;
+          throw new IOException("fake disk full while writing BitVector");
+        }
+      }
+    }
+  }
+  
+  // LUCENE-2593
+  public void testCorruptionAfterDiskFullDuringMerge() throws IOException {
+    MockRAMDirectory dir = new MockRAMDirectory();
+    dir.setPreventDoubleWrite(false);
+    IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED);
+    w.setMergeScheduler(new SerialMergeScheduler());
+
+    ((LogMergePolicy) w.getMergePolicy()).setMergeFactor(2);
+
+    Document doc = new Document();
+    doc.add(new Field("f", "doctor who", Field.Store.YES, Field.Index.ANALYZED));
+    w.addDocument(doc);
+
+    w.commit();
+
+    w.deleteDocuments(new Term("f", "who"));
+    w.addDocument(doc);
+    
+    // disk fills up!
+    FailTwiceDuringMerge ftdm = new FailTwiceDuringMerge();
+    ftdm.setDoFail();
+    dir.failOn(ftdm);
+
+    try {
+      w.commit();
+      fail("fake disk full IOExceptions not hit");
+    } catch (IOException ioe) {
+      // expected
+      assertTrue(ftdm.didFail1 || ftdm.didFail2);
+    }
+    _TestUtil.checkIndex(dir);
+    ftdm.clearDoFail();
+    w.addDocument(doc);
+    w.close();
+
+    _TestUtil.checkIndex(dir);
+    dir.close();
+  }
 }

Modified: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=984217&r1=984216&r2=984217&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
(original)
+++ lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
Tue Aug 10 21:24:02 2010
@@ -29,6 +29,7 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 
 public class TestIndexWriterDelete extends LuceneTestCase {
 
@@ -532,8 +533,10 @@ public class TestIndexWriterDelete exten
 
         // If the close() succeeded, make sure there are
         // no unreferenced files.
-        if (success)
+        if (success) {
+          _TestUtil.checkIndex(dir);
           TestIndexWriter.assertNoUnreferencedFiles(dir, "after writer.close");
+        }
 
         // Finally, verify index is not corrupt, and, if
         // we succeeded, we see all docs changed, and if

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Aug 10 21:24:02 2010
@@ -1,5 +1,5 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:941394,946651,948430,957490
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:941394,946651,948430,957490,984210
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336



Mime
View raw message