lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r770417 - in /lucene/java/branches/lucene_2_4: CHANGES.txt src/java/org/apache/lucene/index/IndexWriter.java
Date Thu, 30 Apr 2009 19:57:37 GMT
Author: mikemccand
Date: Thu Apr 30 19:57:36 2009
New Revision: 770417

URL: http://svn.apache.org/viewvc?rev=770417&view=rev
Log:
LUCENE-1611 (on 2.4 branch): fix case where OutOfMemoryException in IndexWriter could cause
infinite merging to happen

Modified:
    lucene/java/branches/lucene_2_4/CHANGES.txt
    lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/index/IndexWriter.java

Modified: lucene/java/branches/lucene_2_4/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4/CHANGES.txt?rev=770417&r1=770416&r2=770417&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_4/CHANGES.txt Thu Apr 30 19:57:36 2009
@@ -5,6 +5,10 @@
 
 Bug fixes
 
+1. LUCENE-1611: Fix case where OutOfMemoryException in IndexWriter
+   could cause "infinite merging" to happen.  (Christiaan Fluit via
+   Mike McCandless)
+
 ======================= Release 2.4.1 2009-03-09 =======================
 
 Bug fixes

Modified: lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/index/IndexWriter.java?rev=770417&r1=770416&r2=770417&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/index/IndexWriter.java Thu
Apr 30 19:57:36 2009
@@ -1635,7 +1635,9 @@
 
       // Only allow a new merge to be triggered if we are
       // going to wait for merges:
-      flush(waitForMerges, true, true);
+      if (!hitOOM) {
+        flush(waitForMerges, true, true);
+      }
 
       if (waitForMerges)
         // Give merge scheduler last chance to run, in case
@@ -1651,7 +1653,9 @@
       if (infoStream != null)
         message("now call final commit()");
       
-      commit(0);
+      if (!hitOOM) {
+        commit(0);
+      }
 
       if (infoStream != null)
         message("at close: " + segString());
@@ -1672,8 +1676,7 @@
         closed = true;
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "closeInternal");
     } finally {
       synchronized(this) {
         closing = false;
@@ -1936,8 +1939,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addDocument");
     }
   }
 
@@ -1954,8 +1956,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "deleteDocuments");
     }
   }
 
@@ -1974,8 +1975,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "deleteDocuments");
     }
   }
 
@@ -2064,8 +2064,7 @@
       if (doFlush)
         flush(true, false, false);
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "updateDocument");
     }
   }
 
@@ -2263,6 +2262,11 @@
     if (doWait) {
       synchronized(this) {
         while(true) {
+
+          if (hitOOM) {
+            throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot
complete optimize");
+          }
+
           if (mergeExceptions.size() > 0) {
             // Forward any exceptions in background merge
             // threads to the current thread:
@@ -2345,6 +2349,10 @@
         boolean running = true;
         while(running) {
 
+          if (hitOOM) {
+            throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot
complete expungeDeletes");
+          }
+
           // Check each merge that MergePolicy asked us to
           // do, to see if any of them are still running and
           // if any of them have hit an exception.
@@ -2424,6 +2432,11 @@
     if (stopMerges)
       return;
 
+    // Do not start new merges if we've hit OOME
+    if (hitOOM) {
+      return;
+    }
+
     final MergePolicy.MergeSpecification spec;
     if (optimize) {
       spec = mergePolicy.findMergesForOptimize(segmentInfos, this, maxNumSegmentsOptimize,
segmentsToOptimize);
@@ -2731,8 +2744,7 @@
 
       success = true;
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "rollbackInternal");
     } finally {
       synchronized(this) {
         if (!success) {
@@ -2904,8 +2916,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexes");
     } finally {
       docWriter.resumeAllThreads();
     }
@@ -3041,8 +3052,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexesNoOptimize");
     } finally {
       docWriter.resumeAllThreads();
     }
@@ -3296,8 +3306,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "addIndexes");
     } finally {
       docWriter.resumeAllThreads();
     }
@@ -3455,6 +3464,18 @@
   // synchronized, ie, merges should be allowed to commit
   // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
+    try {
+      return doFlushInternal(flushDocStores, flushDeletes);
+    } finally {
+      docWriter.clearFlushPending();
+    }
+  }
+
+  private synchronized final boolean doFlushInternal(boolean flushDocStores, boolean flushDeletes)
throws CorruptIndexException, IOException {
+
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot flush");
+    }
 
     ensureOpen(false);
 
@@ -3605,10 +3626,9 @@
       return flushDocs;
 
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "doFlush");
+      return false;
     } finally {
-      docWriter.clearFlushPending();
       docWriter.resumeAllThreads();
     }
   }
@@ -3757,8 +3777,9 @@
 
     assert testPoint("startCommitMerge");
 
-    if (hitOOM)
-      return false;
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot complete
merge");
+    }
 
     if (infoStream != null)
       message("commitMerge: " + merge.segString(directory) + " index=" + segString());
@@ -3837,6 +3858,11 @@
   }
 
   final private void handleMergeException(Throwable t, MergePolicy.OneMerge merge) throws
IOException {
+
+    if (infoStream != null) {
+      message("handleMergeException: merge=" + merge.segString(directory) + " exc=" + t);
+    }
+
     // Set the exception on the merge, so if
     // optimize() is waiting on us it sees the root
     // cause exception:
@@ -3910,8 +3936,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "merge");
     }
   }
 
@@ -3987,6 +4012,10 @@
     assert merge.registerDone;
     assert !merge.optimize || merge.maxNumSegmentsOptimize > 0;
 
+    if (hitOOM) {
+      throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot merge");
+    }
+
     if (merge.info != null)
       // mergeInit already done
       return;
@@ -4670,8 +4699,7 @@
         }
       }
     } catch (OutOfMemoryError oom) {
-      hitOOM = true;
-      throw oom;
+      handleOOM(oom, "startCommit");
     }
     assert testPoint("finishStartCommit");
   }
@@ -4763,6 +4791,14 @@
         = new MaxFieldLength("LIMITED", DEFAULT_MAX_FIELD_LENGTH);
   }
 
+  private void handleOOM(OutOfMemoryError oom, String location) {
+    if (infoStream != null) {
+      message("hit OutOfMemoryError inside " + location);
+    }
+    hitOOM = true;
+    throw oom;
+  }
+
   // Used only by assert for testing.  Current points:
   //   startDoFlush
   //   startCommitMerge



Mime
View raw message