lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r770414 - in /lucene/java/trunk: CHANGES.txt src/java/org/apache/lucene/index/IndexWriter.java
Date Thu, 30 Apr 2009 19:50:34 GMT
Author: mikemccand
Date: Thu Apr 30 19:50:34 2009
New Revision: 770414

URL: http://svn.apache.org/viewvc?rev=770414&view=rev
Log:
LUCENE-1611: fix case where OutOfMemoryException in IndexWriter could cause infinite merging
to happen

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=770414&r1=770413&r2=770414&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Thu Apr 30 19:50:34 2009
@@ -150,6 +150,10 @@
    when loading documents from the index.  (P Eger via Mike
    McCandless)
 
+7. LUCENE-1611: Fix case where OutOfMemoryException in IndexWriter
+   could cause "infinite merging" to happen.  (Christiaan Fluit via
+   Mike McCandless)
+
 New features
 
  1. LUCENE-1411: Added expert API to open an IndexWriter on a prior

Modified: lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java?rev=770414&r1=770413&r2=770414&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/index/IndexWriter.java Thu Apr 30 19:50:34
2009
@@ -2043,7 +2043,9 @@
 
       // Only allow a new merge to be triggered if we are
       // going to wait for merges:
-      flush(waitForMerges, true, true);
+      if (!hitOOM) {
+        flush(waitForMerges, true, true);
+      }
 
       if (waitForMerges)
         // Give merge scheduler last chance to run, in case
@@ -2059,7 +2061,9 @@
       if (infoStream != null)
         message("now call final commit()");
       
-      commit(0);
+      if (!hitOOM) {
+        commit(0);
+      }
 
       if (infoStream != null)
         message("at close: " + segString());
@@ -2081,8 +2085,7 @@
         closed = true;
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "closeInternal");
     } finally {
       synchronized(this) {
         closing = false;
@@ -2356,8 +2359,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addDocument");
     }
   }
 
@@ -2379,8 +2381,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "deleteDocuments(Term)");
     }
   }
 
@@ -2404,8 +2405,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "deleteDocuments(Term[])");
     }
   }
 
@@ -2514,8 +2514,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "updateDocument");
     }
   }
 
@@ -2708,6 +2707,11 @@
     if (doWait) {
       synchronized(this) {
         while(true) {
+
+          if (hitOOM) {
+            throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot
complete optimize");
+          }
+
           if (mergeExceptions.size() > 0) {
             // Forward any exceptions in background merge
             // threads to the current thread:
@@ -2795,6 +2799,10 @@
         boolean running = true;
         while(running) {
 
+          if (hitOOM) {
+            throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot
complete expungeDeletes");
+          }
+
           // Check each merge that MergePolicy asked us to
           // do, to see if any of them are still running and
           // if any of them have hit an exception.
@@ -2883,6 +2891,11 @@
     if (stopMerges)
       return;
 
+    // Do not start new merges if we've hit OOME
+    if (hitOOM) {
+      return;
+    }
+
     final MergePolicy.MergeSpecification spec;
     if (optimize) {
       spec = mergePolicy.findMergesForOptimize(segmentInfos, this, maxNumSegmentsOptimize,
segmentsToOptimize);
@@ -3196,8 +3209,7 @@
 
       success = true;
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "rollbackInternal");
     } finally {
       synchronized(this) {
         if (!success) {
@@ -3375,8 +3387,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexes(Directory[])");
     } finally {
       if (docWriter != null) {
         docWriter.resumeAllThreads();
@@ -3518,8 +3529,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexesNoOptimize");
     } finally {
       if (docWriter != null) {
         docWriter.resumeAllThreads();
@@ -3761,8 +3771,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexes(IndexReader[])");
     } finally {
       if (docWriter != null) {
         docWriter.resumeAllThreads();
@@ -3794,8 +3803,9 @@
    * @throws IOException if there is a low-level IO error
    */
   public final void flush() throws CorruptIndexException, IOException {  
-    if (hitOOM)
+    if (hitOOM) {
       throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush");
+    }
 
     flush(true, false, true);
   }
@@ -3850,8 +3860,9 @@
 
   private final void prepareCommit(String commitUserData, boolean internal) throws CorruptIndexException,
IOException {
 
-    if (hitOOM)
+    if (hitOOM) {
       throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot commit");
+    }
 
     if (autoCommit && !internal)
       throw new IllegalStateException("this method can only be used when autoCommit is false");
@@ -3980,6 +3991,21 @@
   // synchronized, ie, merges should be allowed to commit
   // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
+    try {
+      return doFlushInternal(flushDocStores, flushDeletes);
+    } finally {
+      docWriter.clearFlushPending();
+    }
+  }
+
+  // TODO: this method should not have to be entirely
+  // synchronized, ie, merges should be allowed to commit
+  // even while a flush is happening
+  private synchronized final boolean doFlushInternal(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
+
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush");
+    }
 
     ensureOpen(false);
 
@@ -4133,10 +4159,10 @@
       return flushDocs;
 
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "doFlush");
+      // never hit
+      return false;
     } finally {
-      docWriter.clearFlushPending();
       docWriter.resumeAllThreads();
     }
   }
@@ -4260,8 +4286,9 @@
 
     assert testPoint("startCommitMerge");
 
-    if (hitOOM)
-      return false;
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot complete
merge");
+    }
 
     if (infoStream != null)
       message("commitMerge: " + merge.segString(directory) + " index=" + segString());
@@ -4401,8 +4428,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "merge");
     }
   }
 
@@ -4477,6 +4503,10 @@
     assert merge.registerDone;
     assert !merge.optimize || merge.maxNumSegmentsOptimize > 0;
 
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot merge");
+    }
+
     if (merge.info != null)
       // mergeInit already done
       return;
@@ -5095,8 +5125,9 @@
 
     assert testPoint("startStartCommit");
 
-    if (hitOOM)
-      return;
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot commit");
+    }
 
     try {
 
@@ -5273,8 +5304,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "startCommit");
     }
     assert testPoint("finishStartCommit");
   }
@@ -5397,6 +5427,14 @@
     return mergedSegmentWarmer;
   }
 
+  private void handleOOM(OutOfMemoryError oom, String location) {
+    if (infoStream != null) {
+      message("hit OutOfMemoryError inside " + location);
+    }
+    hitOOM = true;
+    throw oom;
+  }
+
   // Used only by assert for testing.  Current points:
   //   startDoFlush
   //   startCommitMerge



Mime
View raw message