lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject [Lucene.Net] svn commit: r1082321 [2/4] - in /incubator/lucene.net/trunk/C#/src: Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/ Lucene.Net/Index/ Lucene.Net/Search/ Lucene.Net/Store/ Lucene.Net/Util/ Test/Index/ Test/Search/ Test/Util/
Date Wed, 16 Mar 2011 22:14:43 GMT
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexWriter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs Wed Mar 16 22:14:41 2011
@@ -304,7 +304,8 @@ namespace Lucene.Net.Index
 		private bool autoCommit = true; // false if we should commit only on close
 		
 		private SegmentInfos segmentInfos = new SegmentInfos(); // the segments
-		
+        private int optimizeMaxNumSegments;
+
 		private DocumentsWriter docWriter;
 		private IndexFileDeleter deleter;
 
@@ -340,6 +341,8 @@ namespace Lucene.Net.Index
 		private SupportClass.ThreadClass writeThread; // non-null if any thread holds write lock
 		internal ReaderPool readerPool;
 		private int upgradeCount;
+
+        private int readerTermsIndexDivisor = IndexReader.DEFAULT_TERMS_INDEX_DIVISOR;
 		
 		// This is a "write once" variable (like the organic dye
 		// on a DVD-R that may or may not be heated by a laser and
@@ -414,7 +417,7 @@ namespace Lucene.Net.Index
 		/// <throws>  IOException </throws>
 		public virtual IndexReader GetReader()
 		{
-			return GetReader(IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+            return GetReader(readerTermsIndexDivisor);
 		}
 		
 		/// <summary>Expert: like {@link #getReader}, except you can
@@ -434,6 +437,8 @@ namespace Lucene.Net.Index
 		/// </param>
 		public virtual IndexReader GetReader(int termInfosIndexDivisor)
 		{
+            EnsureOpen();
+
 			if (infoStream != null)
 			{
 				Message("flush at getReader");
@@ -444,16 +449,17 @@ namespace Lucene.Net.Index
 			// this method is called:
 			poolReaders = true;
 			
-			Flush(true, true, false);
-			
 			// Prevent segmentInfos from changing while opening the
 			// reader; in theory we could do similar retry logic,
 			// just like we do when loading segments_N
+            IndexReader r;
 			lock (this)
 			{
-                ApplyDeletes();
-				return new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor);
+                Flush(false, true, true);
+                r = new ReadOnlyDirectoryReader(this, segmentInfos, termInfosIndexDivisor);
 			}
+            MaybeMerge();
+            return r;
 		}
 		
 		/// <summary>Holds shared SegmentReader instances. IndexWriter uses
@@ -568,46 +574,44 @@ namespace Lucene.Net.Index
 				{
 					
 					bool pooled = readerMap.Contains(sr.GetSegmentInfo());
-					
-					System.Diagnostics.Debug.Assert(!pooled | readerMap[sr.GetSegmentInfo()] == sr);
-					
-					// Drop caller's ref
+
+                    System.Diagnostics.Debug.Assert(!pooled || readerMap[sr.GetSegmentInfo()] == sr);
+
+                    // Drop caller's ref; for an external reader (not
+                    // pooled), this decRef will close it
 					sr.DecRef();
 					
 					if (pooled && (drop || (!Enclosing_Instance.poolReaders && sr.GetRefCount() == 1)))
 					{
-						
-						// We are the last ref to this reader; since we're
-						// not pooling readers, we release it:
-						readerMap.Remove(sr.GetSegmentInfo());
+
+                        // We invoke deleter.checkpoint below, so we must be
+                        // sync'd on IW if there are changes:
 						
 						// TODO: java 5
 						// assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this);
+
+                        // Discard (don't save) changes when we are dropping
+                        // the reader; this is used only on the sub-readers
+                        // after a successful merge.
+                        sr.hasChanges &= !drop;
+
+                        bool hasChanges = sr.hasChanges;
 						
 						// Drop our ref -- this will commit any pending
 						// changes to the dir
-						bool success = false;
-						try
-						{
-							sr.Close();
-							success = true;
-						}
-						finally
-						{
-							if (!success && sr.hasChanges)
-							{
-								// Abandon the changes & retry closing:
-								sr.hasChanges = false;
-								try
-								{
-									sr.Close();
-								}
-								catch (System.Exception ignore)
-								{
-									// Keep throwing original exception
-								}
-							}
-						}
+                        sr.Close();
+
+                        // We are the last ref to this reader; since we're
+                        // not pooling readers, we release it:
+                        readerMap.Remove(sr.GetSegmentInfo());
+
+                        if (hasChanges)
+                        {
+                            // Must checkpoint w/ deleter, because this
+                            // segment reader will have created new _X_N.del
+                            // file.
+                            enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
+                        }
 					}
 				}
 			}
@@ -628,23 +632,14 @@ namespace Lucene.Net.Index
 						if (sr.hasChanges)
 						{
 							System.Diagnostics.Debug.Assert(InfoIsLive(sr.GetSegmentInfo()));
-							sr.StartCommit();
-							bool success = false;
-							try
-							{
-								sr.DoCommit(null);
-								success = true;
-							}
-							finally
-							{
-								if (!success)
-								{
-									sr.RollbackCommit();
-								}
-							}
+							sr.DoCommit(null);
+                            // Must checkpoint w/ deleter, because this
+                            // segment reader will have created new _X_N.del
+                            // file.
+                            enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
 						}
-						
-						readerMap.Remove(iter.Current);
+
+                        readerMap.Remove(iter.Current); // iter.remove(); DIGY
 						
 						// NOTE: it is allowed that this decRef does not
 						// actually close the SR; this can happen when a
@@ -670,20 +665,11 @@ namespace Lucene.Net.Index
 						if (sr.hasChanges)
 						{
 							System.Diagnostics.Debug.Assert(InfoIsLive(sr.GetSegmentInfo()));
-							sr.StartCommit();
-							bool success = false;
-							try
-							{
-								sr.DoCommit(null);
-								success = true;
-							}
-							finally
-							{
-								if (!success)
-								{
-									sr.RollbackCommit();
-								}
-							}
+							sr.DoCommit(null);
+                            // Must checkpoint w/ deleter, because this
+                            // segment reader will have created new _X_N.del
+                            // file.
+                            enclosingInstance.deleter.Checkpoint(enclosingInstance.segmentInfos, false);
 						}
 					}
 				}
@@ -723,7 +709,7 @@ namespace Lucene.Net.Index
 			{
 				lock (this)
 				{
-					return Get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+                    return Get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, enclosingInstance.readerTermsIndexDivisor);
 				}
 			}
 			/// <summary> Obtain a SegmentReader from the readerPool.  The reader
@@ -758,7 +744,11 @@ namespace Lucene.Net.Index
 						// synchronized
 						// Returns a ref, which we xfer to readerMap:
 						sr = SegmentReader.Get(info, readBufferSize, doOpenStores, termsIndexDivisor);
-						readerMap[info] = sr;
+                        if (info.dir == enclosingInstance.directory)
+                        {
+                            // Only pool if reader is not external
+                            readerMap[info]=sr;
+                        }
 					}
 					else
 					{
@@ -779,7 +769,11 @@ namespace Lucene.Net.Index
 					}
 					
 					// Return a ref to our caller
-					sr.IncRef();
+                    if (info.dir == enclosingInstance.directory)
+                    {
+                        // Only incRef if we pooled (reader is not external)
+                        sr.IncRef();
+                    }
 					return sr;
 				}
 			}
@@ -932,7 +926,7 @@ namespace Lucene.Net.Index
 		public virtual void  Message(System.String message)
 		{
 			if (infoStream != null)
-				infoStream.WriteLine("IW " + messageID + " [" + SupportClass.ThreadClass.Current().Name + "]: " + message);
+                infoStream.WriteLine("IW " + messageID + " [" + DateTime.Now.ToString() + "; " + SupportClass.ThreadClass.Current().Name + "]: " + message);
 		}
 		
 		private void  SetMessageID(System.IO.StreamWriter infoStream)
@@ -1830,7 +1824,8 @@ namespace Lucene.Net.Index
 				throw new LockObtainFailedException("Index locked for write: " + writeLock);
 			}
 			this.writeLock = writeLock; // save it
-			
+
+            bool success = false;
 			try
 			{
 				if (create)
@@ -1902,7 +1897,7 @@ namespace Lucene.Net.Index
 				
 				// Default deleter (for backwards compatibility) is
 				// KeepOnlyLastCommitDeleter:
-				deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter);
+				deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter,synced);
 				
 				if (deleter.startingCommitDeleted)
 				// Deletion policy deleted the "head" commit point.
@@ -1918,12 +1913,27 @@ namespace Lucene.Net.Index
 					Message("init: create=" + create);
 					MessageState();
 				}
+
+                success = true;
 			}
-			catch (System.IO.IOException e)
+			finally
 			{
-				this.writeLock.Release();
-				this.writeLock = null;
-				throw e;
+                if (!success)
+                {
+                    if (infoStream != null)
+                    {
+                        Message("init: hit exception on init; releasing write lock");
+                    }
+                    try
+                    {
+                        writeLock.Release();
+                    }
+                    catch (Exception t)
+                    {
+                        // don't mask the original exception
+                    }
+                    writeLock = null;
+                }
 			}
 		}
 		
@@ -2072,6 +2082,32 @@ namespace Lucene.Net.Index
 			EnsureOpen();
 			return maxFieldLength;
 		}
+
+        /** Sets the termsIndexDivisor passed to any readers that
+        *  IndexWriter opens, for example when applying deletes
+        *  or creating a near-real-time reader in {@link
+        *  IndexWriter#getReader}.  Default value is {@link
+        *  IndexReader#DEFAULT_TERMS_INDEX_DIVISOR}. */
+        public void SetReaderTermsIndexDivisor(int divisor)
+        {
+            EnsureOpen();
+            if (divisor <= 0)
+            {
+                throw new System.ArgumentException("divisor must be >= 1 (got " + divisor + ")");
+            }
+            readerTermsIndexDivisor = divisor;
+            if (infoStream != null)
+            {
+                Message("setReaderTermsIndexDivisor " + readerTermsIndexDivisor);
+            }
+        }
+
+        /** @see #setReaderTermsIndexDivisor */
+        public int GetReaderTermsIndexDivisor()
+        {
+            EnsureOpen();
+            return readerTermsIndexDivisor;
+        }
 		
 		/// <summary>Determines the minimal number of documents required
 		/// before the buffered in-memory documents are flushed as
@@ -2596,9 +2632,17 @@ namespace Lucene.Net.Index
 		{
 			lock (this)
 			{
-				
+                if (infoStream != null)
+                {
+                    Message("flushDocStores segment=" + docWriter.GetDocStoreSegment());
+                }
+
 				bool useCompoundDocStore = false;
-				
+                if (infoStream != null)
+                {
+                    Message("closeDocStores segment=" + docWriter.GetDocStoreSegment());
+                }
+
 				System.String docStoreSegment;
 				
 				bool success = false;
@@ -2614,7 +2658,12 @@ namespace Lucene.Net.Index
 						Message("hit exception closing doc store segment");
 					}
 				}
-				
+
+                if (infoStream != null)
+                {
+                    Message("flushDocStores files=" + docWriter.ClosedFiles());
+                }
+
 				useCompoundDocStore = mergePolicy.UseCompoundDocStore(segmentInfos);
 				
 				if (useCompoundDocStore && docStoreSegment != null && docWriter.ClosedFiles().Count != 0)
@@ -3151,9 +3200,10 @@ namespace Lucene.Net.Index
 		/// <p/>See http://www.gossamer-threads.com/lists/lucene/java-dev/47895 for more discussion. <p/>
 		/// 
 		/// <p/>Note that optimize requires 2X the index size free
-		/// space in your Directory.  For example, if your index
+		/// space in your Directory (3X if you're using compound
+        /// file format).  For example, if your index
 		/// size is 10 MB then you need 20 MB free for optimize to
-		/// complete.<p/>
+        /// complete (30 MB if you're using compound fiel format).<p/>
 		/// 
 		/// <p/>If some but not all readers re-open while an
 		/// optimize is underway, this will cause > 2X temporary
@@ -3258,6 +3308,7 @@ namespace Lucene.Net.Index
 			{
 				ResetMergeExceptions();
 				segmentsToOptimize = new System.Collections.Hashtable();
+                optimizeMaxNumSegments = maxNumSegments;
 				int numSegments = segmentInfos.Count;
 				for (int i = 0; i < numSegments; i++)
 					SupportClass.CollectionsHelper.AddIfNotContains(segmentsToOptimize, segmentInfos.Info(i));
@@ -3498,9 +3549,11 @@ namespace Lucene.Net.Index
 			lock (this)
 			{
 				System.Diagnostics.Debug.Assert(!optimize || maxNumSegmentsOptimize > 0);
-				
-				if (stopMerges)
-					return ;
+
+                if (stopMerges)
+                {
+                    return;
+                }
 				
 				// Do not start new merges if we've hit OOME
 				if (hitOOM)
@@ -3829,6 +3882,11 @@ namespace Lucene.Net.Index
 		{
 			
 			bool success = false;
+
+            if (infoStream != null)
+            {
+                Message("rollback");
+            }
 			
 			docWriter.PauseAllThreads();
 			
@@ -4600,9 +4658,11 @@ namespace Lucene.Net.Index
 						}
 						finally
 						{
-							
-							deleter.DecRef(files);
-							
+                            lock (this)
+                            {
+                                deleter.DecRef(files);
+                            }
+														
 							if (!success)
 							{
 								if (infoStream != null)
@@ -4630,10 +4690,12 @@ namespace Lucene.Net.Index
 				}
 			}
 		}
-		
-		// This is called after pending added and deleted
-		// documents have been flushed to the Directory but before
-		// the change is committed (new segments_N file written).
+
+        ///<summary>
+        /// A hook for extending classes to execute operations after pending added and
+        /// deleted documents have been flushed to the Directory but before the change
+        /// is committed (new segments_N file written).
+        ///</summary>   
 		protected  virtual void  DoAfterFlush()
 		{
 		}
@@ -4664,6 +4726,14 @@ namespace Lucene.Net.Index
 			
 			Flush(true, false, true);
 		}
+
+        ///<summary>
+        /// A hook for extending classes to execute operations before pending added and
+        /// deleted documents are flushed to the Directory.
+        ///</summary>
+        protected virtual void DoBeforeFlush() 
+        {
+        }
 		
 		/// <summary>Expert: prepare for commit.
 		/// 
@@ -4855,11 +4925,15 @@ namespace Lucene.Net.Index
 						System.Threading.Monitor.PulseAll(this);
 					}
 				}
-				else if (infoStream != null)
-					Message("commit: pendingCommit == null; skip");
-				
-				if (infoStream != null)
-					Message("commit: done");
+                else if (infoStream != null)
+                {
+                    Message("commit: pendingCommit == null; skip");
+                }
+
+                if (infoStream != null)
+                {
+                    Message("commit: done");
+                }
 			}
 		}
 		
@@ -4896,6 +4970,10 @@ namespace Lucene.Net.Index
 				}
 				finally
 				{
+                    if (docWriter.DoBalanceRAM())
+                    {
+                        docWriter.BalanceRAM();
+                    }
 					docWriter.ClearFlushPending();
 				}
 			}
@@ -4917,6 +4995,8 @@ namespace Lucene.Net.Index
 				EnsureOpen(false);
 				
 				System.Diagnostics.Debug.Assert(TestPoint("startDoFlush"));
+
+                DoBeforeFlush();
 				
 				flushCount++;
 				
@@ -4934,6 +5014,10 @@ namespace Lucene.Net.Index
 				// Make sure no threads are actively adding a document.
 				// Returns true if docWriter is currently aborting, in
 				// which case we skip flushing this segment
+                if (infoStream != null)
+                {
+                    Message("flush: now pause all indexing threads");
+                }
 				if (docWriter.PauseAllThreads())
 				{
 					docWriter.ResumeAllThreads();
@@ -5001,6 +5085,10 @@ namespace Lucene.Net.Index
 						try
 						{
 							flushedDocCount = docWriter.Flush(flushDocStores);
+                            if (infoStream != null)
+                            {
+                                Message("flushedFiles=" + docWriter.GetFlushedFiles());
+                            }
 							success = true;
 						}
 						finally
@@ -5226,7 +5314,7 @@ namespace Lucene.Net.Index
 				
 				System.Diagnostics.Debug.Assert(mergeReader.NumDeletedDocs() == delCount);
 				
-				mergeReader.hasChanges = delCount >= 0;
+				mergeReader.hasChanges = delCount > 0;
 			}
 		}
 		
@@ -5259,7 +5347,6 @@ namespace Lucene.Net.Index
 					if (infoStream != null)
 						Message("commitMerge: skipping merge " + merge.SegString(directory) + ": it was aborted");
 					
-					deleter.Refresh(merge.info.name);
 					return false;
 				}
 				
@@ -5267,32 +5354,20 @@ namespace Lucene.Net.Index
 				
 				CommitMergedDeletes(merge, mergedReader);
 				docWriter.RemapDeletes(segmentInfos, merger.GetDocMaps(), merger.GetDelCounts(), merge, mergedDocCount);
-				
-				// Simple optimization: if the doc store we are using
-				// has been closed and is in now compound format (but
-				// wasn't when we started), then we will switch to the
-				// compound format as well:
-				System.String mergeDocStoreSegment = merge.info.GetDocStoreSegment();
-				if (mergeDocStoreSegment != null && !merge.info.GetDocStoreIsCompoundFile())
-				{
-					int size = segmentInfos.Count;
-					for (int i = 0; i < size; i++)
-					{
-						SegmentInfo info = segmentInfos.Info(i);
-						System.String docStoreSegment = info.GetDocStoreSegment();
-						if (docStoreSegment != null && docStoreSegment.Equals(mergeDocStoreSegment) && info.GetDocStoreIsCompoundFile())
-						{
-							merge.info.SetDocStoreIsCompoundFile(true);
-							break;
-						}
-					}
-				}
+
+                // If the doc store we are using has been closed and
+                // is in now compound format (but wasn't when we
+                // started), then we will switch to the compound
+                // format as well:
+                SetMergeDocStoreIsCompoundFile(merge);
 				
 				merge.info.SetHasProx(merger.HasProx());
 				
 				((System.Collections.IList) ((System.Collections.ArrayList) segmentInfos).GetRange(start, start + merge.segments.Count - start)).Clear();
 				System.Diagnostics.Debug.Assert(!segmentInfos.Contains(merge.info));
 				segmentInfos.Insert(start, merge.info);
+
+                CloseMergeReaders(merge, false);
 				
 				// Must note the change to segmentInfos so any commits
 				// in-flight don't lose it:
@@ -5302,22 +5377,16 @@ namespace Lucene.Net.Index
 				// them so that they don't bother writing them to
 				// disk, updating SegmentInfo, etc.:
 				readerPool.Clear(merge.segments);
-				
-				if (merge.optimize)
+
+                if (merge.optimize)
+                {
+                    // cascade the optimize:
                     segmentsToOptimize[merge.info] = merge.info;
+                }
 				return true;
 			}
 		}
 		
-		private void  DecrefMergeSegments(MergePolicy.OneMerge merge)
-		{
-			lock (this)
-			{
-				System.Diagnostics.Debug.Assert(merge.increfDone);
-				merge.increfDone = false;
-			}
-		}
-		
 		private void  HandleMergeException(System.Exception t, MergePolicy.OneMerge merge)
 		{
 			
@@ -5444,12 +5513,23 @@ namespace Lucene.Net.Index
 				for (int i = 0; i < count; i++)
 				{
 					SegmentInfo info = merge.segments.Info(i);
-					if (mergingSegments.Contains(info))
-						return false;
-					if (segmentInfos.IndexOf(info) == - 1)
-						return false;
-					if (info.dir != directory)
-						isExternal = true;
+                    if (mergingSegments.Contains(info))
+                    {
+                        return false;
+                    }
+                    if (segmentInfos.IndexOf(info) == -1)
+                    {
+                        return false;
+                    }
+                    if (info.dir != directory)
+                    {
+                        isExternal = true;
+                    }
+                    if (segmentsToOptimize.Contains(info))
+                    {
+                        merge.optimize = true;
+                        merge.maxNumSegmentsOptimize = optimizeMaxNumSegments;
+                    }
 				}
 				
 				EnsureContiguousMerge(merge);
@@ -5591,7 +5671,15 @@ namespace Lucene.Net.Index
 						doFlushDocStore = true;
 					}
 				}
-				
+
+                // if a mergedSegmentWarmer is installed, we must merge
+                // the doc stores because we will open a full
+                // SegmentReader on the merged segment:
+                if (!mergeDocStores && mergedSegmentWarmer != null && currentDocStoreSegment != null && lastDocStoreSegment != null && lastDocStoreSegment.Equals(currentDocStoreSegment))
+                {
+                    mergeDocStores = true;
+                }
+
 				int docStoreOffset;
 				System.String docStoreSegment2;
 				bool docStoreIsCompoundFile;
@@ -5625,8 +5713,6 @@ namespace Lucene.Net.Index
 					DoFlush(true, false);
 				}
 				
-				merge.increfDone = true;
-				
 				merge.mergeDocStores = mergeDocStores;
 				
 				// Bind a new segment name here so even with
@@ -5736,9 +5822,6 @@ namespace Lucene.Net.Index
 				// on merges to finish.
 				System.Threading.Monitor.PulseAll(this);
 				
-				if (merge.increfDone)
-					DecrefMergeSegments(merge);
-				
 				// It's possible we are called twice, eg if there was an
 				// exception inside mergeInit
 				if (merge.registerDone)
@@ -5756,6 +5839,92 @@ namespace Lucene.Net.Index
 			}
 		}
 		
+        private void SetMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge)
+        {
+            lock (this)
+            {
+                string mergeDocStoreSegment = merge.info.GetDocStoreSegment();
+                if (mergeDocStoreSegment != null && !merge.info.GetDocStoreIsCompoundFile())
+                {
+                    int size = segmentInfos.Count;
+                    for (int i = 0; i < size; i++)
+                    {
+                        SegmentInfo info = segmentInfos.Info(i);
+                        string docStoreSegment = info.GetDocStoreSegment();
+                        if (docStoreSegment != null &&
+                            docStoreSegment.Equals(mergeDocStoreSegment) &&
+                            info.GetDocStoreIsCompoundFile())
+                        {
+                            merge.info.SetDocStoreIsCompoundFile(true);
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+
+        private void CloseMergeReaders(MergePolicy.OneMerge merge, bool suppressExceptions)
+        {
+            lock (this)
+            {
+                int numSegments = merge.segments.Count;
+                if (suppressExceptions)
+                {
+                    // Suppress any new exceptions so we throw the
+                    // original cause
+                    for (int i = 0; i < numSegments; i++)
+                    {
+                        if (merge.readers[i] != null)
+                        {
+                            try
+                            {
+                                readerPool.Release(merge.readers[i], false);
+                            }
+                            catch (Exception t)
+                            {
+                            }
+                            merge.readers[i] = null;
+                        }
+
+                        if (merge.readersClone[i] != null)
+                        {
+                            try
+                            {
+                                merge.readersClone[i].Close();
+                            }
+                            catch (Exception t)
+                            {
+                            }
+                            // This was a private clone and we had the
+                            // only reference
+                            System.Diagnostics.Debug.Assert(merge.readersClone[i].GetRefCount() == 0); //: "refCount should be 0 but is " + merge.readersClone[i].getRefCount();
+                            merge.readersClone[i] = null;
+                        }
+                    }
+                }
+                else
+                {
+                    for (int i = 0; i < numSegments; i++)
+                    {
+                        if (merge.readers[i] != null)
+                        {
+                            readerPool.Release(merge.readers[i], true);
+                            merge.readers[i] = null;
+                        }
+
+                        if (merge.readersClone[i] != null)
+                        {
+                            merge.readersClone[i].Close();
+                            // This was a private clone and we had the only reference
+                            System.Diagnostics.Debug.Assert(merge.readersClone[i].GetRefCount() == 0);
+                            merge.readersClone[i] = null;
+                        }
+                    }
+                }
+            }
+        }
+
+
 		/// <summary>Does the actual (time-consuming) work of the merge,
 		/// but without holding synchronized lock on IndexWriter
 		/// instance 
@@ -5786,250 +5955,229 @@ namespace Lucene.Net.Index
 
             System.Collections.Hashtable dss = new System.Collections.Hashtable();
 			
+            String currentDocStoreSegment;
+            lock(this) {
+                currentDocStoreSegment = docWriter.GetDocStoreSegment();
+            }
+            bool currentDSSMerged = false;
+
 			// This is try/finally to make sure merger's readers are
 			// closed:
 			bool success = false;
-			try
-			{
-				int totDocCount = 0;
-				
-				for (int i = 0; i < numSegments; i++)
-				{
-					
-					SegmentInfo info = sourceSegments.Info(i);
-					
-					// Hold onto the "live" reader; we will use this to
-					// commit merged deletes
-					SegmentReader reader = merge.readers[i] = readerPool.Get(info, merge.mergeDocStores, MERGE_READ_BUFFER_SIZE, - 1);
-					
-					// We clone the segment readers because other
-					// deletes may come in while we're merging so we
-					// need readers that will not change
-					SegmentReader clone = merge.readersClone[i] = (SegmentReader) reader.Clone(true);
-					merger.Add(clone);
-					
-					if (clone.HasDeletions())
-					{
-						mergeDocStores = true;
-					}
-					
-					if (info.GetDocStoreOffset() != - 1)
-					{
-                        dss[info.GetDocStoreSegment()] = info.GetDocStoreSegment();
-					}
-					
-					totDocCount += clone.NumDocs();
-				}
-				
-				if (infoStream != null)
-				{
-					Message("merge: total " + totDocCount + " docs");
-				}
-				
-				merge.CheckAborted(directory);
-				
-				// If deletions have arrived and it has now become
-				// necessary to merge doc stores, go and open them:
-				if (mergeDocStores && !merge.mergeDocStores)
-				{
-					merge.mergeDocStores = true;
-					lock (this)
-					{
-                        String key = docWriter.GetDocStoreSegment();
-						if (key!=null && dss.Contains(key))
-						{
-							if (infoStream != null)
-								Message("now flush at mergeMiddle");
-							DoFlush(true, false);
-						}
-					}
-					
-					for (int i = 0; i < numSegments; i++)
-					{
-						merge.readersClone[i].OpenDocStores();
-					}
-					
-					// Clear DSS
-					lock (this)
-					{
-						merge.info.SetDocStore(- 1, null, false);
-					}
-				}
-				
-				// This is where all the work happens:
-				mergedDocCount = merge.info.docCount = merger.Merge(merge.mergeDocStores);
-				
-				System.Diagnostics.Debug.Assert(mergedDocCount == totDocCount);
-				
-				// TODO: in the non-realtime case, we may want to only
-				// keep deletes (it's costly to open entire reader
-				// when we just need deletes)
-				
-				SegmentReader mergedReader = readerPool.Get(merge.info, false, BufferedIndexInput.BUFFER_SIZE, - 1);
-				try
-				{
-					if (poolReaders && mergedSegmentWarmer != null)
-					{
-						mergedSegmentWarmer.Warm(mergedReader);
-					}
-					if (!CommitMerge(merge, merger, mergedDocCount, mergedReader))
-					// commitMerge will return false if this merge was aborted
-						return 0;
-				}
-				finally
-				{
-					lock (this)
-					{
-						readerPool.Release(mergedReader);
-					}
-				}
-				
-				success = true;
-			}
-			finally
-			{
-				lock (this)
-				{
-					if (!success)
-					{
-						// Suppress any new exceptions so we throw the
-						// original cause
-						for (int i = 0; i < numSegments; i++)
-						{
-							if (merge.readers[i] != null)
-							{
-								try
-								{
-									readerPool.Release(merge.readers[i], true);
-								}
-								catch (System.Exception t)
-								{
-								}
-							}
-							
-							if (merge.readersClone[i] != null)
-							{
-								try
-								{
-									merge.readersClone[i].Close();
-								}
-								catch (System.Exception t)
-								{
-								}
-								// This was a private clone and we had the only reference
-								System.Diagnostics.Debug.Assert(merge.readersClone[i].GetRefCount() == 0);
-							}
-						}
-					}
-					else
-					{
-						for (int i = 0; i < numSegments; i++)
-						{
-							if (merge.readers[i] != null)
-							{
-								readerPool.Release(merge.readers[i], true);
-							}
-							
-							if (merge.readersClone[i] != null)
-							{
-								merge.readersClone[i].Close();
-								// This was a private clone and we had the only reference
-								System.Diagnostics.Debug.Assert(merge.readersClone[i].GetRefCount() == 0);
-							}
-						}
-					}
-				}
-			}
-			
-			// Must checkpoint before decrefing so any newly
-			// referenced files in the new merge.info are incref'd
-			// first:
-			lock (this)
-			{
-				deleter.Checkpoint(segmentInfos, false);
-			}
-			DecrefMergeSegments(merge);
-			
-			if (merge.useCompoundFile)
-			{
-				
-				// Maybe force a sync here to allow reclaiming of the
-				// disk space used by the segments we just merged:
-				if (autoCommit && DoCommitBeforeMergeCFS(merge))
-				{
-					long size;
-					lock (this)
-					{
-						size = merge.info.SizeInBytes();
-					}
-					Commit(size);
-				}
-				
-				success = false;
-				System.String compoundFileName = mergedName + "." + IndexFileNames.COMPOUND_FILE_EXTENSION;
-				
-				try
-				{
-					merger.CreateCompoundFile(compoundFileName);
-					success = true;
-				}
-				catch (System.IO.IOException ioe)
-				{
-					lock (this)
-					{
-						if (merge.IsAborted())
-						{
-							// This can happen if rollback or close(false)
-							// is called -- fall through to logic below to
-							// remove the partially created CFS:
-							success = true;
-						}
-						else
-							HandleMergeException(ioe, merge);
-					}
-				}
-				catch (System.Exception t)
-				{
-					HandleMergeException(t, merge);
-				}
-				finally
-				{
-					if (!success)
-					{
-						if (infoStream != null)
-							Message("hit exception creating compound file during merge");
-						lock (this)
-						{
-							deleter.DeleteFile(compoundFileName);
-						}
-					}
-				}
-				
-				if (merge.IsAborted())
-				{
-					if (infoStream != null)
-						Message("abort merge after building CFS");
-					deleter.DeleteFile(compoundFileName);
-					return 0;
-				}
-				
-				lock (this)
-				{
-					if (segmentInfos.IndexOf(merge.info) == - 1 || merge.IsAborted())
-					{
-						// Our segment (committed in non-compound
-						// format) got merged away while we were
-						// building the compound format.
-						deleter.DeleteFile(compoundFileName);
-					}
-					else
-					{
-						merge.info.SetUseCompoundFile(true);
-						Checkpoint();
-					}
-				}
-			}
-			
+            try
+            {
+                int totDocCount = 0;
+
+                for (int i = 0; i < numSegments; i++)
+                {
+
+                    SegmentInfo info = sourceSegments.Info(i);
+
+                    // Hold onto the "live" reader; we will use this to
+                    // commit merged deletes
+                    SegmentReader reader = merge.readers[i] = readerPool.Get(info, merge.mergeDocStores, MERGE_READ_BUFFER_SIZE, -1);
+
+                    // We clone the segment readers because other
+                    // deletes may come in while we're merging so we
+                    // need readers that will not change
+                    SegmentReader clone = merge.readersClone[i] = (SegmentReader)reader.Clone(true);
+                    merger.Add(clone);
+
+                    if (clone.HasDeletions())
+                    {
+                        mergeDocStores = true;
+                    }
+
+                    if (info.GetDocStoreOffset() != -1 && currentDocStoreSegment != null)
+                    {
+                        currentDSSMerged |= currentDocStoreSegment.Equals(info.GetDocStoreSegment());
+                    }
+
+                    totDocCount += clone.NumDocs();
+                }
+
+                if (infoStream != null)
+                {
+                    Message("merge: total " + totDocCount + " docs");
+                }
+
+                merge.CheckAborted(directory);
+
+                // If deletions have arrived and it has now become
+                // necessary to merge doc stores, go and open them:
+                if (mergeDocStores && !merge.mergeDocStores)
+                {
+                    merge.mergeDocStores = true;
+                    lock (this)
+                    {
+                        if (currentDSSMerged)
+                        {
+                            if (infoStream != null)
+                            {
+                                Message("now flush at mergeMiddle");
+                            }
+                            DoFlush(true, false);
+                        }
+                    }
+
+                    for (int i = 0; i < numSegments; i++)
+                    {
+                        merge.readersClone[i].OpenDocStores();
+                    }
+
+                    // Clear DSS
+                    merge.info.SetDocStore(-1, null, false);
+
+                }
+
+                // This is where all the work happens:
+                mergedDocCount = merge.info.docCount = merger.Merge(merge.mergeDocStores);
+
+                System.Diagnostics.Debug.Assert(mergedDocCount == totDocCount);
+                /////////////////////////////////////// start DIGY
+
+                if (merge.useCompoundFile)
+                {
+
+                    success = false;
+                    string compoundFileName = IndexFileNames.SegmentFileName(mergedName, IndexFileNames.COMPOUND_FILE_EXTENSION);
+
+                    try
+                    {
+                        if (infoStream != null)
+                        {
+                            Message("create compound file " + compoundFileName);
+                        }
+                        merger.CreateCompoundFile(compoundFileName);
+                        success = true;
+                    }
+                    catch (System.IO.IOException ioe)
+                    {
+                        lock (this)
+                        {
+                            if (merge.IsAborted())
+                            {
+                                // This can happen if rollback or close(false)
+                                // is called -- fall through to logic below to
+                                // remove the partially created CFS:
+                            }
+                            else
+                            {
+                                HandleMergeException(ioe, merge);
+                            }
+                        }
+                    }
+                    catch (Exception t)
+                    {
+                        HandleMergeException(t, merge);
+                    }
+                    finally
+                    {
+                        if (!success)
+                        {
+                            if (infoStream != null)
+                            {
+                                Message("hit exception creating compound file during merge");
+                            }
+
+                            lock (this)
+                            {
+                                deleter.DeleteFile(compoundFileName);
+                                deleter.DeleteNewFiles(merger.GetMergedFiles());
+                            }
+                        }
+                    }
+
+                    success = false;
+
+                    lock (this)
+                    {
+
+                        // delete new non cfs files directly: they were never
+                        // registered with IFD
+                        deleter.DeleteNewFiles(merger.GetMergedFiles());
+
+                        if (merge.IsAborted())
+                        {
+                            if (infoStream != null)
+                            {
+                                Message("abort merge after building CFS");
+                            }
+                            deleter.DeleteFile(compoundFileName);
+                            return 0;
+                        }
+                    }
+
+                    merge.info.SetUseCompoundFile(true);
+                }
+
+                int termsIndexDivisor;
+                bool loadDocStores;
+
+                // if the merged segment warmer was not installed when
+                // this merge was started, causing us to not force
+                // the docStores to close, we can't warm it now
+                bool canWarm = merge.info.GetDocStoreSegment() == null || currentDocStoreSegment == null || !merge.info.GetDocStoreSegment().Equals(currentDocStoreSegment);
+
+                if (poolReaders && mergedSegmentWarmer != null && canWarm)
+                {
+                    // Load terms index & doc stores so the segment
+                    // warmer can run searches, load documents/term
+                    // vectors
+                    termsIndexDivisor = readerTermsIndexDivisor;
+                    loadDocStores = true;
+                }
+                else
+                {
+                    termsIndexDivisor = -1;
+                    loadDocStores = false;
+                }
+
+                // TODO: in the non-realtime case, we may want to only
+                // keep deletes (it's costly to open entire reader
+                // when we just need deletes)
+
+                SegmentReader mergedReader = readerPool.Get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor);
+                try
+                {
+                    if (poolReaders && mergedSegmentWarmer != null)
+                    {
+                        mergedSegmentWarmer.Warm(mergedReader);
+                    }
+                    if (!CommitMerge(merge, merger, mergedDocCount, mergedReader))
+                    {
+                        // commitMerge will return false if this merge was aborted
+                        return 0;
+                    }
+                }
+                finally
+                {
+                    lock (this)
+                    {
+                        readerPool.Release(mergedReader);
+                    }
+                }
+
+                success = true;
+            }
+            finally
+            {
+                // Readers are already closed in commitMerge if we didn't hit
+                // an exc:
+                if (!success)
+                {
+                    CloseMergeReaders(merge, true);
+                }
+            }
+
+            merge.mergeDone = true;
+
+            lock (mergeScheduler)
+            {
+                System.Threading.Monitor.PulseAll(mergeScheduler); 
+            }
+
+			///////////////////////////////////////////////////////// end DIGY
 			// Force a sync after commiting the merge.  Once this
 			// sync completes then all index files referenced by the
 			// current segmentInfos are on stable storage so if the
@@ -6068,7 +6216,7 @@ namespace Lucene.Net.Index
 			{
 				System.Diagnostics.Debug.Assert(TestPoint("startApplyDeletes"));
                 flushDeletesCount++;
-				SegmentInfos rollback = (SegmentInfos) segmentInfos.Clone();
+				
 				bool success = false;
 				bool changed;
 				try
@@ -6078,28 +6226,10 @@ namespace Lucene.Net.Index
 				}
 				finally
 				{
-					if (!success)
-					{
-						if (infoStream != null)
-							Message("hit exception flushing deletes");
-						
-						// Carefully remove any partially written .del
-						// files
-						int size = rollback.Count;
-						for (int i = 0; i < size; i++)
-						{
-							System.String newDelFileName = segmentInfos.Info(i).GetDelFileName();
-							System.String delFileName = rollback.Info(i).GetDelFileName();
-							if (newDelFileName != null && !newDelFileName.Equals(delFileName))
-								deleter.DeleteFile(newDelFileName);
-						}
-						
-						// Fully replace the segmentInfos since flushed
-						// deletes could have changed any of the
-						// SegmentInfo instances:
-						segmentInfos.Clear();
-						segmentInfos.AddRange(rollback);
-					}
+                    if (!success && infoStream != null)
+                    {
+                        Message("hit exception flushing deletes");
+                    }
 				}
 				
 				if (changed)
@@ -6129,7 +6259,7 @@ namespace Lucene.Net.Index
 		// utility routines for tests
 		public /*internal*/ virtual SegmentInfo NewestSegment()
 		{
-			return segmentInfos.Info(segmentInfos.Count - 1);
+            return segmentInfos.Count > 0 ? segmentInfos.Info(segmentInfos.Count - 1) : null;
 		}
 		
 		public virtual System.String SegString()
@@ -6355,6 +6485,7 @@ namespace Lucene.Net.Index
 					{
 						
 						System.Diagnostics.Debug.Assert(lastCommitChangeCount <= changeCount);
+                        myChangeCount = changeCount;
 						
 						if (changeCount == lastCommitChangeCount)
 						{
@@ -6372,7 +6503,27 @@ namespace Lucene.Net.Index
 						if (infoStream != null)
 							Message("startCommit index=" + SegString(segmentInfos) + " changeCount=" + changeCount);
 						
-						readerPool.Commit();
+						// It's possible another flush (that did not close
+                        // the open do stores) snuck in after the flush we
+                        // just did, so we remove any tail segments
+                        // referencing the open doc store from the
+                        // SegmentInfos we are about to sync (the main
+                        // SegmentInfos will keep them):
+                        toSync = (SegmentInfos) segmentInfos.Clone();
+                        string dss = docWriter.GetDocStoreSegment();
+                        if (dss != null)
+                        {
+                            while (true)
+                            {
+                                String dss2 = toSync.Info(toSync.Count - 1).GetDocStoreSegment();
+                                if (dss2 == null || !dss2.Equals(dss))
+                                {
+                                    break;
+                                }
+                                toSync.Remove(toSync.Count - 1);
+                                changeCount++;
+                            }
+                        }
 						
 						toSync = (SegmentInfos) segmentInfos.Clone();
 						
@@ -6380,13 +6531,18 @@ namespace Lucene.Net.Index
 							toSync.SetUserData(commitUserData);
 						
 						deleter.IncRef(toSync, false);
-						myChangeCount = changeCount;
-						
+												
 						System.Collections.Generic.IEnumerator<string> it = toSync.Files(directory, false).GetEnumerator();
 						while (it.MoveNext())
 						{
 							System.String fileName = it.Current;
 							System.Diagnostics.Debug.Assert(directory.FileExists(fileName), "file " + fileName + " does not exist");
+                            // If this trips it means we are missing a call to
+                            // .checkpoint somewhere, because by the time we
+                            // are called, deleter should know about every
+                            // file referenced by the current head
+                            // segmentInfos:
+                            System.Diagnostics.Debug.Assert(deleter.Exists(fileName));
 						}
 					}
 					finally
@@ -6724,11 +6880,17 @@ namespace Lucene.Net.Index
 					// if any structural changes (new segments), we are
 					// stale
 					return false;
-				}
-				else
-				{
-					return !docWriter.AnyChanges();
-				}
+                }
+                else if (infos.GetGeneration() != segmentInfos.GetGeneration())
+                {
+                    // if any commit took place since we were opened, we
+                    // are stale
+                    return false;
+                }
+                else
+                {
+                    return !docWriter.AnyChanges();
+                }
 			}
 		}
 		

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/LogMergePolicy.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs Wed Mar 16 22:14:41 2011
@@ -57,12 +57,20 @@ namespace Lucene.Net.Index
 		/// <seealso cref="setMaxMergeDocs">
 		/// </seealso>
 		public static readonly int DEFAULT_MAX_MERGE_DOCS = System.Int32.MaxValue;
+
+        /// <summary> Default noCFSRatio.  If a merge's size is >= 10% of
+        ///  the index, then we disable compound file for it.
+        ///  @see #setNoCFSRatio 
+        ///  </summary>
+        public static double DEFAULT_NO_CFS_RATIO = 0.1;
 		
 		private int mergeFactor = DEFAULT_MERGE_FACTOR;
 		
 		internal long minMergeSize;
 		internal long maxMergeSize;
 		internal int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
+
+        protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
 		
 		/* TODO 3.0: change this default to true */
 		protected internal bool calibrateSizeByDeletes = false;
@@ -78,6 +86,27 @@ namespace Lucene.Net.Index
 		{
 			return writer != null && writer.Verbose();
 		}
+
+
+        /** @see #setNoCFSRatio */
+        public double GetNoCFSRatio()
+        {
+            return noCFSRatio;
+        }
+
+        /** If a merged segment will be more than this percentage
+         *  of the total size of the index, leave the segment as
+         *  non-compound file even if compound file is enabled.
+         *  Set to 1.0 to always use CFS regardless of merge
+         *  size. */
+        public void SetNoCFSRatio(double noCFSRatio)
+        {
+            if (noCFSRatio < 0.0 || noCFSRatio > 1.0)
+            {
+                throw new ArgumentException("noCFSRatio must be 0.0 to 1.0 inclusive; got " + noCFSRatio);
+            }
+            this.noCFSRatio = noCFSRatio;
+        }
 		
 		private void  Message(System.String message)
 		{
@@ -233,7 +262,8 @@ namespace Lucene.Net.Index
 		private bool IsOptimized(SegmentInfo info)
 		{
 			bool hasDeletions = writer.NumDeletedDocs(info) > 0;
-			return !hasDeletions && !info.HasSeparateNorms() && info.dir == writer.GetDirectory() && info.GetUseCompoundFile() == useCompoundFile;
+			return !hasDeletions && !info.HasSeparateNorms() && info.dir == writer.GetDirectory() &&
+                (info.GetUseCompoundFile() == useCompoundFile || noCFSRatio < 1.0);
 		}
 		
 		/// <summary>Returns the merges necessary to optimize the index.
@@ -277,7 +307,7 @@ namespace Lucene.Net.Index
 					// mergeFactor) to potentially be run concurrently:
 					while (last - maxNumSegments + 1 >= mergeFactor)
 					{
-						spec.Add(new OneMerge(infos.Range(last - mergeFactor, last), useCompoundFile));
+                        spec.Add(MakeOneMerge(infos, infos.Range(last - mergeFactor, last)));
 						last -= mergeFactor;
 					}
 					
@@ -291,7 +321,7 @@ namespace Lucene.Net.Index
 							// Since we must optimize down to 1 segment, the
 							// choice is simple:
 							if (last > 1 || !IsOptimized(infos.Info(0)))
-								spec.Add(new OneMerge(infos.Range(0, last), useCompoundFile));
+                                spec.Add(MakeOneMerge(infos, infos.Range(0, last)));
 						}
 						else if (last > maxNumSegments)
 						{
@@ -322,8 +352,8 @@ namespace Lucene.Net.Index
 									bestSize = sumSize;
 								}
 							}
-							
-							spec.Add(new OneMerge(infos.Range(bestStart, bestStart + finalMergeSize), useCompoundFile));
+
+                            spec.Add(MakeOneMerge(infos, infos.Range(bestStart, bestStart + finalMergeSize)));
 						}
 					}
 				}
@@ -365,7 +395,7 @@ namespace Lucene.Net.Index
 						// deletions, so force a merge now:
 						if (Verbose())
 							Message("  add merge " + firstSegmentWithDeletions + " to " + (i - 1) + " inclusive");
-						spec.Add(new OneMerge(segmentInfos.Range(firstSegmentWithDeletions, i), useCompoundFile));
+                        spec.Add(MakeOneMerge(segmentInfos, segmentInfos.Range(firstSegmentWithDeletions, i)));
 						firstSegmentWithDeletions = i;
 					}
 				}
@@ -376,7 +406,7 @@ namespace Lucene.Net.Index
 					// mergeFactor segments
 					if (Verbose())
 						Message("  add merge " + firstSegmentWithDeletions + " to " + (i - 1) + " inclusive");
-					spec.Add(new OneMerge(segmentInfos.Range(firstSegmentWithDeletions, i), useCompoundFile));
+                    spec.Add(MakeOneMerge(segmentInfos, segmentInfos.Range(firstSegmentWithDeletions, i)));
 					firstSegmentWithDeletions = - 1;
 				}
 			}
@@ -385,7 +415,7 @@ namespace Lucene.Net.Index
 			{
 				if (Verbose())
 					Message("  add merge " + firstSegmentWithDeletions + " to " + (numSegments - 1) + " inclusive");
-				spec.Add(new OneMerge(segmentInfos.Range(firstSegmentWithDeletions, numSegments), useCompoundFile));
+                spec.Add(MakeOneMerge(segmentInfos, segmentInfos.Range(firstSegmentWithDeletions, numSegments)));
 			}
 			
 			return spec;
@@ -497,7 +527,7 @@ namespace Lucene.Net.Index
 							spec = new MergeSpecification();
 						if (Verbose())
 							Message("    " + start + " to " + end + ": add this merge");
-						spec.Add(new OneMerge(infos.Range(start, end), useCompoundFile));
+                        spec.Add(MakeOneMerge(infos, infos.Range(start, end)));
 					}
 					else if (Verbose())
 						Message("    " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
@@ -511,6 +541,36 @@ namespace Lucene.Net.Index
 			
 			return spec;
 		}
+        
+        protected OneMerge MakeOneMerge(SegmentInfos infos, SegmentInfos infosToMerge)
+        {
+            bool doCFS;
+            if (!useCompoundFile)
+            {
+                doCFS = false;
+            }
+            else if (noCFSRatio == 1.0)
+            {
+                doCFS = true;
+            }
+            else
+            {
+                long totSize = 0;
+                for (int i = 0; i < infos.Count; i++)
+                {
+                    totSize += Size(infos.Info(i));
+                }
+                long mergeSize = 0;
+                for (int i = 0; i < infosToMerge.Count; i++)
+                {
+                    mergeSize += Size(infosToMerge.Info(i));
+                }
+
+                doCFS = mergeSize <= noCFSRatio * totSize;
+            }
+
+            return new OneMerge(infosToMerge, doCFS);
+        }
 		
 		/// <summary><p/>Determines the largest segment (measured by
 		/// document count) that may be merged with other segments.

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/MergePolicy.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs Wed Mar 16 22:14:41 2011
@@ -74,7 +74,6 @@ namespace Lucene.Net.Index
 			internal SegmentInfo info; // used by IndexWriter
 			internal bool mergeDocStores; // used by IndexWriter
 			internal bool optimize; // used by IndexWriter
-			internal bool increfDone; // used by IndexWriter
 			internal bool registerDone; // used by IndexWriter
 			internal long mergeGen; // used by IndexWriter
 			internal bool isExternal; // used by IndexWriter
@@ -85,6 +84,8 @@ namespace Lucene.Net.Index
 			internal bool useCompoundFile;
 			internal bool aborted;
 			internal System.Exception error;
+
+            internal volatile bool mergeDone;     // used by IndexWriter
 			
 			public OneMerge(SegmentInfos segments, bool useCompoundFile)
 			{

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/MultiReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs Wed Mar 16 22:14:41 2011
@@ -255,19 +255,18 @@ namespace Lucene.Net.Index
 		
 		public override int NumDocs()
 		{
-			lock (this)
+			// Don't call ensureOpen() here (it could affect performance)
+            // NOTE: multiple threads may wind up init'ing
+            // numDocs... but that's harmless
+			if (numDocs == - 1)
 			{
-				// Don't call ensureOpen() here (it could affect performance)
-				if (numDocs == - 1)
-				{
-					// check cache
-					int n = 0; // cache miss--recompute
-					for (int i = 0; i < subReaders.Length; i++)
-						n += subReaders[i].NumDocs(); // sum from readers
-					numDocs = n;
-				}
-				return numDocs;
+				// check cache
+				int n = 0; // cache miss--recompute
+				for (int i = 0; i < subReaders.Length; i++)
+					n += subReaders[i].NumDocs(); // sum from readers
+				numDocs = n;
 			}
+			return numDocs;
 		}
 		
 		public override int MaxDoc()
@@ -464,6 +463,11 @@ namespace Lucene.Net.Index
 					}
 				}
 			}
+
+            // NOTE: only needed in case someone had asked for
+            // FieldCache for top-level reader (which is generally
+            // not a good idea):
+            Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(this);
 		}
 
         public override System.Collections.Generic.ICollection<string> GetFieldNames(IndexReader.FieldOption fieldNames)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ParallelReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs Wed Mar 16 22:14:41 2011
@@ -534,6 +534,8 @@ namespace Lucene.Net.Index
 					}
 				}
 			}
+
+            Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(this);
 		}
 
         public override System.Collections.Generic.ICollection<string> GetFieldNames(IndexReader.FieldOption fieldNames)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ReusableStringReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs Wed Mar 16 22:14:41 2011
@@ -50,6 +50,7 @@ namespace Lucene.Net.Index
             }
             else if (0 == left)
             {
+                s = null;
                 return 0;
             }
             else

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentInfos.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs Wed Mar 16 22:14:41 2011
@@ -505,8 +505,16 @@ namespace Lucene.Net.Index
 		/// <throws>  IOException if there is a low-level IO error </throws>
 		public static long ReadCurrentVersion(Directory directory)
 		{
-			
-			return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run());
+            // Fully read the segments file: this ensures that it's
+            // completely written so that if
+            // IndexWriter.prepareCommit has been called (but not
+            // yet commit), then the reader will still see itself as
+            // current:
+            SegmentInfos sis = new SegmentInfos();
+            sis.Read(directory);
+            return sis.version;
+			//return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run());
+            //DIGY: AnonymousClassFindSegmentsFile1 can safely be deleted
 		}
 		
 		/// <summary> Returns userData from latest segments file</summary>
@@ -677,7 +685,7 @@ namespace Lucene.Net.Index
 						
 						// Method 2: open segments.gen and read its
 						// contents.  Then we take the larger of the two
-						// gen's.  This way, if either approach is hitting
+						// gens.  This way, if either approach is hitting
 						// a stale cache (NFS) we have a better chance of
 						// getting the right generation.
 						long genB = - 1;
@@ -814,10 +822,8 @@ namespace Lucene.Net.Index
 					try
 					{
 						System.Object v = DoBody(segmentFileName);
-						if (exc != null)
-						{
-							Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
-						}
+						Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
+						
 						return v;
 					}
 					catch (System.IO.IOException err)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentMerger.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs Wed Mar 16 22:14:41 2011
@@ -253,12 +253,10 @@ namespace Lucene.Net.Index
 				((IndexReader) iter.Current).Close();
 			}
 		}
-		
-		public /*internal*/ System.Collections.IList CreateCompoundFile(System.String fileName)
+
+        public /*internal*/ System.Collections.Generic.ICollection<string> GetMergedFiles()
 		{
-			CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
-			
-			System.Collections.IList files = new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + 1);
+            System.Collections.Generic.IDictionary<string,string> fileSet = new System.Collections.Generic.Dictionary<string,string>();
 			
 			// Basic files
 			for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
@@ -269,7 +267,7 @@ namespace Lucene.Net.Index
 					continue;
 				
 				if (mergeDocStores || (!ext.Equals(IndexFileNames.FIELDS_EXTENSION) && !ext.Equals(IndexFileNames.FIELDS_INDEX_EXTENSION)))
-					files.Add(segment + "." + ext);
+                    fileSet[segment + "." + ext] = segment + "." + ext;
 			}
 			
 			// Fieldable norm files
@@ -278,7 +276,7 @@ namespace Lucene.Net.Index
 				FieldInfo fi = fieldInfos.FieldInfo(i);
 				if (fi.isIndexed && !fi.omitNorms)
 				{
-					files.Add(segment + "." + IndexFileNames.NORMS_EXTENSION);
+                    fileSet[segment + "." + IndexFileNames.NORMS_EXTENSION]=segment + "." + IndexFileNames.NORMS_EXTENSION;
 					break;
 				}
 			}
@@ -288,10 +286,18 @@ namespace Lucene.Net.Index
 			{
 				for (int i = 0; i < IndexFileNames.VECTOR_EXTENSIONS.Length; i++)
 				{
-					files.Add(segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]);
+                    fileSet[segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i]] = segment + "." + IndexFileNames.VECTOR_EXTENSIONS[i];
 				}
 			}
-			
+
+            return fileSet.Keys;
+        }
+
+        public /*internal*/ System.Collections.Generic.ICollection<string> CreateCompoundFile(System.String fileName)
+        {
+            System.Collections.Generic.ICollection<string> files = GetMergedFiles();
+            CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
+
 			// Now merge all added files
 			System.Collections.IEnumerator it = files.GetEnumerator();
 			while (it.MoveNext())
@@ -301,8 +307,8 @@ namespace Lucene.Net.Index
 			
 			// Perform the merge
 			cfsWriter.Close();
-			
-			return files;
+
+            return files;
 		}
 
         private void AddIndexed(IndexReader reader, FieldInfos fInfos, System.Collections.Generic.ICollection<string> names, bool storeTermVectors, bool storePositionWithTermVector, bool storeOffsetWithTermVector, bool storePayloads, bool omitTFAndPositions)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs Wed Mar 16 22:14:41 2011
@@ -62,6 +62,7 @@ namespace Lucene.Net.Index
 		private bool rollbackHasChanges = false;
 		private bool rollbackDeletedDocsDirty = false;
 		private bool rollbackNormsDirty = false;
+        private SegmentInfo rollbackSegmentInfo;
 		private int rollbackPendingDeleteCount;
 		
 		// optionally used for the .nrm file shared by multiple norms
@@ -93,14 +94,16 @@ namespace Lucene.Net.Index
 			internal Directory cfsDir;
 			internal int readBufferSize;
 			internal int termsIndexDivisor;
+
+            internal SegmentReader origInstance;
 			
 			internal TermInfosReader tis;
 			internal FieldsReader fieldsReaderOrig;
 			internal TermVectorsReader termVectorsReaderOrig;
 			internal CompoundFileReader cfsReader;
 			internal CompoundFileReader storeCFSReader;
-			
-			internal CoreReaders(Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
+
+            internal CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor)
 			{
 				segment = si.name;
 				this.readBufferSize = readBufferSize;
@@ -153,6 +156,13 @@ namespace Lucene.Net.Index
 						DecRef();
 					}
 				}
+
+
+                // Must assign this at the end -- if we hit an
+                // exception above core, we don't want to attempt to
+                // purge the FieldCache (will hit NPE because core is
+                // not assigned yet).
+                this.origInstance = origInstance;
 			}
 			
 			internal TermVectorsReader GetTermVectorsReaderOrig()
@@ -294,6 +304,12 @@ namespace Lucene.Net.Index
 						{
 							storeCFSReader.Close();
 						}
+
+                        // Force FieldCache to evict our entries at this point
+                        if (origInstance != null)
+                        {
+                            Lucene.Net.Search.FieldCache_Fields.DEFAULT.Purge(origInstance);
+                        }
 					}
 				}
 			}
@@ -721,14 +737,32 @@ namespace Lucene.Net.Index
 				
 				// NOTE: norms are re-written in regular directory, not cfs
 				si.AdvanceNormGen(this.number);
-				IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
+				string normFileName = si.GetNormFileName(this.number);
+                IndexOutput @out = enclosingInstance.Directory().CreateOutput(normFileName);
+                bool success = false;
 				try
 				{
-					out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
+					try {
+                        @out.WriteBytes(bytes, enclosingInstance.MaxDoc());
+                    } finally {
+                        @out.Close();
+                    }
+                    success = true;
 				}
 				finally
 				{
-					out_Renamed.Close();
+                    if (!success)
+                    {
+                        try
+                        {
+                            enclosingInstance.Directory().DeleteFile(normFileName);
+                        }
+                        catch (Exception t)
+                        {
+                            // suppress this so we keep throwing the
+                            // original exception
+                        }
+                    }
 				}
 				this.dirty = false;
 			}
@@ -794,7 +828,7 @@ namespace Lucene.Net.Index
 			
 			try
 			{
-				instance.core = new CoreReaders(dir, si, readBufferSize, termInfosIndexDivisor);
+				instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
 				if (doOpenStores)
 				{
 					instance.core.OpenDocStores(si);
@@ -823,6 +857,21 @@ namespace Lucene.Net.Index
 		{
 			core.OpenDocStores(si);
 		}
+
+        private bool CheckDeletedCounts()
+        {
+            int recomputedCount = deletedDocs.GetRecomputedCount();
+
+            System.Diagnostics.Debug.Assert(deletedDocs.Count() == recomputedCount, "deleted count=" + deletedDocs.Count() + " vs recomputed count=" + recomputedCount);
+
+            System.Diagnostics.Debug.Assert(si.GetDelCount() == recomputedCount, "delete count mismatch: info=" + si.GetDelCount() + " vs BitVector=" + recomputedCount);
+
+            // Verify # deletes does not exceed maxDoc for this
+            // segment:
+            System.Diagnostics.Debug.Assert(si.GetDelCount() <= MaxDoc(), "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + MaxDoc() + ") for segment " + si.name);
+
+            return true;
+        }
 		
 		private void  LoadDeletedDocs()
 		{
@@ -831,14 +880,8 @@ namespace Lucene.Net.Index
 			{
 				deletedDocs = new BitVector(Directory(), si.GetDelFileName());
 				deletedDocsRef = new Ref();
-				
-				System.Diagnostics.Debug.Assert(si.GetDelCount() == deletedDocs.Count(), 
-					"delete count mismatch: info=" + si.GetDelCount() + " vs BitVector=" + deletedDocs.Count());
-				
-				// Verify # deletes does not exceed maxDoc for this
-				// segment:
-				System.Diagnostics.Debug.Assert(si.GetDelCount() <= MaxDoc(), 
-					"delete count mismatch: " + deletedDocs.Count() + ") exceeds max doc (" + MaxDoc() + ") for segment  + si.name");
+
+                System.Diagnostics.Debug.Assert(CheckDeletedCounts());
 			}
 			else 
 				System.Diagnostics.Debug.Assert(si.GetDelCount() == 0);
@@ -1019,48 +1062,86 @@ namespace Lucene.Net.Index
 		}
 
         protected internal override void DoCommit(System.Collections.Generic.IDictionary<string, string> commitUserData)
-		{
-			if (hasChanges)
-			{
-				if (deletedDocsDirty)
-				{
-					// re-write deleted
-					si.AdvanceDelGen();
-					
-					// We can write directly to the actual name (vs to a
-					// .tmp & renaming it) because the file is not live
-					// until segments file is written:
-					deletedDocs.Write(Directory(), si.GetDelFileName());
-					
-					si.SetDelCount(si.GetDelCount() + pendingDeleteCount);
-					pendingDeleteCount = 0;
-					System.Diagnostics.Debug.Assert(deletedDocs.Count() == si.GetDelCount(), "delete count mismatch during commit: info=" + si.GetDelCount() + " vs BitVector=" + deletedDocs.Count());
-				}
-				else
-				{
-					System.Diagnostics.Debug.Assert(pendingDeleteCount == 0);
-				}
-				
-				if (normsDirty)
-				{
-					// re-write norms
-					si.SetNumFields(core.fieldInfos.Size());
-					System.Collections.IEnumerator it = norms.Values.GetEnumerator();
-					while (it.MoveNext())
-					{
-						Norm norm = (Norm) it.Current;
-						if (norm.dirty)
-						{
-							norm.ReWrite(si);
-						}
-					}
-				}
-				deletedDocsDirty = false;
-				normsDirty = false;
-				hasChanges = false;
-			}
-		}
-		
+        {
+            if (hasChanges)
+            {
+                StartCommit();
+                bool success = false;
+                try
+                {
+                    CommitChanges(commitUserData);
+                    success = true;
+                }
+                finally
+                {
+                    if (!success)
+                    {
+                        RollbackCommit();
+                    }
+                }
+            }
+        }
+
+        private void CommitChanges(System.Collections.Generic.IDictionary<string, string> commitUserData)
+        {
+            if (deletedDocsDirty)
+            {               // re-write deleted
+                si.AdvanceDelGen();
+
+                // We can write directly to the actual name (vs to a
+                // .tmp & renaming it) because the file is not live
+                // until segments file is written:
+                string delFileName = si.GetDelFileName();
+                bool success = false;
+                try
+                {
+                    deletedDocs.Write(Directory(), delFileName);
+                    success = true;
+                }
+                finally
+                {
+                    if (!success)
+                    {
+                        try
+                        {
+                            Directory().DeleteFile(delFileName);
+                        }
+                        catch (Exception t)
+                        {
+                            // suppress this so we keep throwing the
+                            // original exception
+                        }
+                    }
+                }
+
+                si.SetDelCount(si.GetDelCount() + pendingDeleteCount);
+                pendingDeleteCount = 0;
+                System.Diagnostics.Debug.Assert(deletedDocs.Count() == si.GetDelCount(), "delete count mismatch during commit: info=" + si.GetDelCount() + " vs BitVector=" + deletedDocs.Count());
+            }
+            else
+            {
+                System.Diagnostics.Debug.Assert(pendingDeleteCount == 0);
+            }
+
+            if (normsDirty)
+            {               // re-write norms
+                si.SetNumFields(core.fieldInfos.Size());
+                System.Collections.IEnumerator it = norms.Values.GetEnumerator();
+                while (it.MoveNext())
+                {
+                    Norm norm = (Norm)it.Current;
+                    if (norm.dirty)
+                    {
+                        norm.ReWrite(si);
+                    }
+                }
+            }
+
+            deletedDocsDirty = false;
+            normsDirty = false;
+            hasChanges = false;
+        }
+        
 		internal virtual FieldsReader GetFieldsReader()
 		{
 			return (FieldsReader) fieldsReaderLocal.Get();
@@ -1603,6 +1684,7 @@ namespace Lucene.Net.Index
 		
 		internal virtual void  StartCommit()
 		{
+            rollbackSegmentInfo = (SegmentInfo)si.Clone();
 			rollbackHasChanges = hasChanges;
 			rollbackDeletedDocsDirty = deletedDocsDirty;
 			rollbackNormsDirty = normsDirty;
@@ -1617,6 +1699,7 @@ namespace Lucene.Net.Index
 		
 		internal virtual void  RollbackCommit()
 		{
+            si.Reset(rollbackSegmentInfo);
 			hasChanges = rollbackHasChanges;
 			deletedDocsDirty = rollbackDeletedDocsDirty;
 			normsDirty = rollbackNormsDirty;
@@ -1646,7 +1729,13 @@ namespace Lucene.Net.Index
 		{
 			return core.freqStream;
 		}
-		
+
+		public override object GetDeletesCacheKey() 
+        {
+            return deletedDocs;
+        }
+
+
 		public override long GetUniqueTermCount()
 		{
 			return core.GetTermsReader().Size();

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs Wed Mar 16 22:14:41 2011
@@ -86,6 +86,10 @@ namespace Lucene.Net.Index
 		{
 			lock (this)
 			{
+                if (lastCommit == null)
+                {
+                    throw new System.SystemException("no index commits to snapshot !");
+                }
 				if (snapshot == null)
 					snapshot = lastCommit.GetSegmentsFileName();
 				else
@@ -127,6 +131,12 @@ namespace Lucene.Net.Index
 				InitBlock(enclosingInstance);
 				this.cp = cp;
 			}
+
+            public override string ToString()
+            {
+                return "SnapshotDeletionPolicy.SnapshotCommitPoint(" + cp + ")";
+            }
+
 			public override System.String GetSegmentsFileName()
 			{
 				return cp.GetSegmentsFileName();
@@ -165,6 +175,11 @@ namespace Lucene.Net.Index
 			{
 				return cp.GetUserData();
 			}
+
+            public override bool IsOptimized()
+            {
+                return cp.IsOptimized();
+            }
 		}
 		
 		private System.Collections.IList WrapCommits(System.Collections.IList commits)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/StoredFieldsWriter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs Wed Mar 16 22:14:41 2011
@@ -222,7 +222,7 @@ namespace Lucene.Net.Index
 			private void  InitBlock(StoredFieldsWriter enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
-                buffer = enclosingInstance.docWriter.newPerDocBuffer();
+                buffer = enclosingInstance.docWriter.NewPerDocBuffer();
                 fdt = new RAMOutputStream(buffer);
 			}
 			private StoredFieldsWriter enclosingInstance;
@@ -234,15 +234,15 @@ namespace Lucene.Net.Index
 				}
 				
 			}
-			
-			internal DocumentsWriter.PerDocBuffer buffer;
-			internal RAMOutputStream fdt;
+
+            internal DocumentsWriter.PerDocBuffer buffer ;
+            internal RAMOutputStream fdt;
 			internal int numStoredFields;
 			
 			internal void  Reset()
 			{
 				fdt.Reset();
-				buffer.recycle();
+                buffer.Recycle();
 				numStoredFields = 0;
 			}
 			
@@ -254,7 +254,7 @@ namespace Lucene.Net.Index
 			
 			public override long SizeInBytes()
 			{
-				return buffer.GetSizeInBytes();
+                return buffer.GetSizeInBytes();
 			}
 			
 			public override void  Finish()

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/TermVectorsReader.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs Wed Mar 16 22:14:41 2011
@@ -76,46 +76,53 @@ namespace Lucene.Net.Index
 			
 			try
 			{
-				if (d.FileExists(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION))
-				{
-					tvx = d.OpenInput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION, readBufferSize);
-					format = CheckValidFormat(tvx);
-					tvd = d.OpenInput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, readBufferSize);
-					int tvdFormat = CheckValidFormat(tvd);
-					tvf = d.OpenInput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION, readBufferSize);
-					int tvfFormat = CheckValidFormat(tvf);
-					
-					System.Diagnostics.Debug.Assert(format == tvdFormat);
-					System.Diagnostics.Debug.Assert(format == tvfFormat);
-					
-					if (format >= FORMAT_VERSION2)
-					{
-						System.Diagnostics.Debug.Assert((tvx.Length() - FORMAT_SIZE) % 16 == 0);
-						numTotalDocs = (int) (tvx.Length() >> 4);
-					}
-					else
-					{
-						System.Diagnostics.Debug.Assert((tvx.Length() - FORMAT_SIZE) % 8 == 0);
-						numTotalDocs = (int) (tvx.Length() >> 3);
-					}
-					
-					if (- 1 == docStoreOffset)
-					{
-						this.docStoreOffset = 0;
-						this.size = numTotalDocs;
-						System.Diagnostics.Debug.Assert(size == 0 || numTotalDocs == size);
-					}
-					else
-					{
-						this.docStoreOffset = docStoreOffset;
-						this.size = size;
-						// Verify the file is long enough to hold all of our
-						// docs
-						System.Diagnostics.Debug.Assert(numTotalDocs >= size + docStoreOffset, "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset);
-					}
-				}
-				else
-					format = 0;
+                if (d.FileExists(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION))
+                {
+                    tvx = d.OpenInput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION, readBufferSize);
+                    format = CheckValidFormat(tvx);
+                    tvd = d.OpenInput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, readBufferSize);
+                    int tvdFormat = CheckValidFormat(tvd);
+                    tvf = d.OpenInput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION, readBufferSize);
+                    int tvfFormat = CheckValidFormat(tvf);
+
+                    System.Diagnostics.Debug.Assert(format == tvdFormat);
+                    System.Diagnostics.Debug.Assert(format == tvfFormat);
+
+                    if (format >= FORMAT_VERSION2)
+                    {
+                        System.Diagnostics.Debug.Assert((tvx.Length() - FORMAT_SIZE) % 16 == 0);
+                        numTotalDocs = (int)(tvx.Length() >> 4);
+                    }
+                    else
+                    {
+                        System.Diagnostics.Debug.Assert((tvx.Length() - FORMAT_SIZE) % 8 == 0);
+                        numTotalDocs = (int)(tvx.Length() >> 3);
+                    }
+
+                    if (-1 == docStoreOffset)
+                    {
+                        this.docStoreOffset = 0;
+                        this.size = numTotalDocs;
+                        System.Diagnostics.Debug.Assert(size == 0 || numTotalDocs == size);
+                    }
+                    else
+                    {
+                        this.docStoreOffset = docStoreOffset;
+                        this.size = size;
+                        // Verify the file is long enough to hold all of our
+                        // docs
+                        System.Diagnostics.Debug.Assert(numTotalDocs >= size + docStoreOffset, "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset);
+                    }
+                }
+                else
+                {
+                    // If all documents flushed in a segment had hit
+                    // non-aborting exceptions, it's possible that
+                    // FieldInfos.hasVectors returns true yet the term
+                    // vector files don't exist.
+                    format = 0;
+                }
+
 				
 				this.fieldInfos = fieldInfos;
 				success = true;

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/TermVectorsTermsWriter.cs?rev=1082321&r1=1082320&r2=1082321&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs Wed Mar 16 22:14:41 2011
@@ -62,7 +62,13 @@ namespace Lucene.Net.Index
 		{
 			lock (this)
 			{
-				
+                // NOTE: it's possible that all documents seen in this segment
+                // hit non-aborting exceptions, in which case we will
+                // not have yet init'd the TermVectorsWriter.  This is
+                // actually OK (unlike in the stored fields case)
+                // because, although IieldInfos.hasVectors() will return
+                // true, the TermVectorsReader gracefully handles
+                // non-existence of the term vectors files.
 				if (tvx != null)
 				{
 					
@@ -231,15 +237,14 @@ namespace Lucene.Net.Index
 						tvd.WriteVLong(pos - lastPos);
 						lastPos = pos;
 					}
-					perDoc.perDocTvf.WriteTo(tvf);
-					perDoc.perDocTvf.Reset();
+                    perDoc.perDocTvf.WriteTo(tvf);
 					perDoc.numVectorFields = 0;
 				}
 				
 				System.Diagnostics.Debug.Assert(lastDocID == perDoc.docID + docWriter.GetDocStoreOffset());
 				
 				lastDocID++;
-				
+                perDoc.Reset();
 				Free(perDoc);
 				System.Diagnostics.Debug.Assert(docWriter.writer.TestPoint("TermVectorsTermsWriter.finishDocument end"));
 			}
@@ -308,8 +313,8 @@ namespace Lucene.Net.Index
 			private void  InitBlock(TermVectorsTermsWriter enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
-                this.buffer = enclosingInstance.docWriter.newPerDocBuffer();
-                this.perDocTvf = new RAMOutputStream(this.buffer);
+                buffer = enclosingInstance.docWriter.NewPerDocBuffer();
+                perDocTvf = new RAMOutputStream(buffer);
 			}
 			private TermVectorsTermsWriter enclosingInstance;
 			public TermVectorsTermsWriter Enclosing_Instance
@@ -322,8 +327,7 @@ namespace Lucene.Net.Index
 			}
 			
 			internal DocumentsWriter.PerDocBuffer buffer;
-			internal RAMOutputStream perDocTvf;
-
+            internal RAMOutputStream perDocTvf;
 			internal int numVectorFields;
 			
 			internal int[] fieldNumbers = new int[1];
@@ -331,8 +335,8 @@ namespace Lucene.Net.Index
 			
 			internal void  Reset()
 			{
-				perDocTvf.Reset();
-				buffer.recycle();
+                perDocTvf.Reset();
+                buffer.Recycle();
 				numVectorFields = 0;
 			}
 			
@@ -350,13 +354,13 @@ namespace Lucene.Net.Index
 					fieldPointers = ArrayUtil.Grow(fieldPointers);
 				}
 				fieldNumbers[numVectorFields] = fieldNumber;
-				fieldPointers[numVectorFields] = perDocTvf.GetFilePointer();
+                fieldPointers[numVectorFields] = perDocTvf.GetFilePointer();
 				numVectorFields++;
 			}
 			
 			public override long SizeInBytes()
 			{
-				return buffer.GetSizeInBytes();
+                return buffer.GetSizeInBytes();
 			}
 			
 			public override void  Finish()



Mime
View raw message