Return-Path: 
 <lucene-net-commits-return-524-apmail-incubator-lucene-net-commits-archive=incubator.apache.org@incubator.apache.org>
Delivered-To: apmail-incubator-lucene-net-commits-archive@minotaur.apache.org
Received: (qmail 68073 invoked from network); 29 Jul 2009 18:05:18 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3)
  by minotaur.apache.org with SMTP; 29 Jul 2009 18:05:18 -0000
Received: (qmail 90348 invoked by uid 500); 29 Jul 2009 18:05:19 -0000
Delivered-To: apmail-incubator-lucene-net-commits-archive@incubator.apache.org
Received: (qmail 90198 invoked by uid 500); 29 Jul 2009 18:05:19 -0000
Mailing-List: contact lucene-net-commits-help@incubator.apache.org;
 run by ezmlm
Precedence: bulk
List-Help: <mailto:lucene-net-commits-help@incubator.apache.org>
List-Unsubscribe: <mailto:lucene-net-commits-unsubscribe@incubator.apache.org>
List-Post: <mailto:lucene-net-commits@incubator.apache.org>
List-Id: <lucene-net-commits.incubator.apache.org>
Reply-To: lucene-net-dev@incubator.apache.org
Delivered-To: mailing list lucene-net-commits@incubator.apache.org
Received: (qmail 90058 invoked by uid 99); 29 Jul 2009 18:05:18 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Jul 2009 18:05:18 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Jul 2009 18:05:00 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 8BC4A23889BB; Wed, 29 Jul 2009 18:04:40 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r798995 [5/35] - in /incubator/lucene.net/trunk/C#/src:
 Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/
 Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/
 Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net/...
Date: Wed, 29 Jul 2009 18:04:24 -0000
To: lucene-net-commits@incubator.apache.org
From: dougsale@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20090729180440.8BC4A23889BB@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs Wed Jul 29 18:04:12 2009
@@ -15,432 +15,458 @@
  * limitations under the License.
  */
 
-using System;
-
 using Directory = Lucene.Net.Store.Directory;
 
 namespace Lucene.Net.Index
 {
-	
-	/// <summary>A {@link MergeScheduler} that runs each merge using a
-	/// separate thread, up until a maximum number of threads
-	/// ({@link #setMaxThreadCount}) at which points merges are
-	/// run in the foreground, serially.  This is a simple way
-	/// to use concurrency in the indexing process without
-	/// having to create and manage application level
-	/// threads. 
-	/// </summary>
-	
-	public class ConcurrentMergeScheduler : MergeScheduler
-	{
-		
-		private int mergeThreadPriority = - 1;
-		
-		private System.Collections.IList mergeThreads = new System.Collections.ArrayList();
-		private int maxThreadCount = 3;
-		
-		private System.Collections.IList exceptions = new System.Collections.ArrayList();
-		private Directory dir;
-		
-		private bool closed;
-		private IndexWriter writer;
-		
-		public ConcurrentMergeScheduler()
-		{
-			if (allInstances != null)
-			{
-				// Only for testing
-				AddMyself();
-			}
-		}
-		
-		/// <summary>Sets the max # simultaneous threads that may be
-		/// running.  If a merge is necessary yet we already have
-		/// this many threads running, the merge is returned back
-		/// to IndexWriter so that it runs in the "foreground". 
-		/// </summary>
-		public virtual void  SetMaxThreadCount(int count)
-		{
-			if (count < 1)
-				throw new System.ArgumentException("count should be at least 1");
-			maxThreadCount = count;
-		}
-		
-		/// <summary>Get the max # simultaneous threads that may be</summary>
-		/// <seealso cref="setMaxThreadCount.">
-		/// </seealso>
-		public virtual int GetMaxThreadCount()
-		{
-			return maxThreadCount;
-		}
-		
-		/// <summary>Return the priority that merge threads run at.  By
-		/// default the priority is 1 plus the priority of (ie,
-		/// slightly higher priority than) the first thread that
-		/// calls merge. 
-		/// </summary>
-		public virtual int GetMergeThreadPriority()
-		{
-			lock (this)
-			{
-				InitMergeThreadPriority();
-				return mergeThreadPriority;
-			}
-		}
-		
-		/// <summary>Return the priority that merge threads run at. </summary>
-		public virtual void  SetMergeThreadPriority(int pri)
-		{
-			lock (this)
-			{
-				if (pri > (int) System.Threading.ThreadPriority.Highest || pri < (int) System.Threading.ThreadPriority.Lowest)
-					throw new System.ArgumentException("priority must be in range " + (int) System.Threading.ThreadPriority.Lowest + " .. " + (int) System.Threading.ThreadPriority.Highest + " inclusive");
-				mergeThreadPriority = pri;
-				
-				int numThreads = MergeThreadCount();
-				for (int i = 0; i < numThreads; i++)
-				{
-					MergeThread merge = (MergeThread) mergeThreads[i];
-					merge.SetThreadPriority(pri);
-				}
-			}
-		}
-		
-		private void  Message(System.String message)
-		{
-			if (writer != null)
-				writer.Message("CMS: " + message);
-		}
-		
-		private void  InitMergeThreadPriority()
-		{
-			lock (this)
-			{
-				if (mergeThreadPriority == - 1)
-				{
-					// Default to slightly higher priority than our
-					// calling thread
-					mergeThreadPriority = 1 + (System.Int32) SupportClass.ThreadClass.Current().Priority;
-					if (mergeThreadPriority > (int) System.Threading.ThreadPriority.Highest)
-						mergeThreadPriority = (int) System.Threading.ThreadPriority.Highest;
-				}
-			}
-		}
-		
-		public override void  Close()
-		{
-			closed = true;
-		}
-		
-		public virtual void  Sync()
-		{
-			lock (this)
-			{
-				while (MergeThreadCount() > 0)
-				{
-					Message("now wait for threads; currently " + mergeThreads.Count + " still running");
-					int count = mergeThreads.Count;
-					for (int i = 0; i < count; i++)
-						Message("    " + i + ": " + ((MergeThread) mergeThreads[i]));
-					
-					try
-					{
-						System.Threading.Monitor.Wait(this);
-					}
-					catch (System.Threading.ThreadInterruptedException)
-					{
-					}
-				}
-			}
-		}
-		
-		private int MergeThreadCount()
-		{
-			lock (this)
-			{
-				int count = 0;
-				int numThreads = mergeThreads.Count;
-				for (int i = 0; i < numThreads; i++)
-					if (((MergeThread) mergeThreads[i]).IsAlive)
-						count++;
-				return count;
-			}
-		}
-		
-		public override void  Merge(IndexWriter writer)
-		{
-			
-			this.writer = writer;
-			
-			InitMergeThreadPriority();
-			
-			dir = writer.GetDirectory();
-			
-			// First, quickly run through the newly proposed merges
-			// and add any orthogonal merges (ie a merge not
-			// involving segments already pending to be merged) to
-			// the queue.  If we are way behind on merging, many of
-			// these newly proposed merges will likely already be
-			// registered.
-			
-			Message("now merge");
-			Message("  index: " + writer.SegString());
-			
-			// Iterate, pulling from the IndexWriter's queue of
-			// pending merges, until its empty:
-			while (true)
-			{
-				
-				// TODO: we could be careful about which merges to do in
-				// the BG (eg maybe the "biggest" ones) vs FG, which
-				// merges to do first (the easiest ones?), etc.
-				
-				MergePolicy.OneMerge merge = writer.GetNextMerge();
-				if (merge == null)
-				{
-					Message("  no more merges pending; now return");
-					return ;
-				}
-				
-				// We do this w/ the primary thread to keep
-				// deterministic assignment of segment names
-				writer.MergeInit(merge);
-				
-				Message("  consider merge " + merge.SegString(dir));
-				
-				if (merge.isExternal)
-				{
-					Message("    merge involves segments from an external directory; now run in foreground");
-				}
-				else
-				{
-					lock (this)
-					{
-						if (MergeThreadCount() < maxThreadCount)
-						{
-							// OK to spawn a new merge thread to handle this
-							// merge:
-							MergeThread merger = new MergeThread(this, writer, merge);
-							mergeThreads.Add(merger);
-							Message("    launch new thread [" + merger.Name + "]");
-							merger.SetThreadPriority(mergeThreadPriority);
-							merger.IsBackground = true;
-							merger.Start();
-							continue;
-						}
-						else
-							Message("    too many merge threads running; run merge in foreground");
-					}
-				}
-				
-				// Too many merge threads already running, so we do
-				// this in the foreground of the calling thread
-				writer.Merge(merge);
-			}
-		}
-		
-		private class MergeThread:SupportClass.ThreadClass
-		{
-			private void  InitBlock(ConcurrentMergeScheduler enclosingInstance)
-			{
-				this.enclosingInstance = enclosingInstance;
-			}
-			private ConcurrentMergeScheduler enclosingInstance;
-			public ConcurrentMergeScheduler Enclosing_Instance
-			{
-				get
-				{
-					return enclosingInstance;
-				}
-				
-			}
-			
-			internal IndexWriter writer;
-			internal MergePolicy.OneMerge startMerge;
-			internal MergePolicy.OneMerge runningMerge;
-			
-			public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
-			{
-				InitBlock(enclosingInstance);
-				this.writer = writer;
-				this.startMerge = startMerge;
-			}
-			
-			public virtual void  SetRunningMerge(MergePolicy.OneMerge merge)
-			{
-				lock (this)
-				{
-					runningMerge = merge;
-				}
-			}
-			
-			public virtual MergePolicy.OneMerge GetRunningMerge()
-			{
-				lock (this)
-				{
-					return runningMerge;
-				}
-			}
-			
-			public virtual void  SetThreadPriority(int pri)
-			{
-				try
-				{
-					Priority = (System.Threading.ThreadPriority) pri;
-				}
-				catch (System.NullReferenceException)
-				{
-					// Strangely, Sun's JDK 1.5 on Linux sometimes
-					// throws NPE out of here...
-				}
-				catch (System.Security.SecurityException)
-				{
-					// Ignore this because we will still run fine with
-					// normal thread priority
-				}
-			}
-			
-			override public void  Run()
-			{
-				
-				// First time through the while loop we do the merge
-				// that we were started with:
-				MergePolicy.OneMerge merge = this.startMerge;
-				
-				try
-				{
-					
-					Enclosing_Instance.Message("  merge thread: start");
-					
-					while (true)
-					{
-						SetRunningMerge(merge);
-						writer.Merge(merge);
-						
-						// Subsequent times through the loop we do any new
-						// merge that writer says is necessary:
-						merge = writer.GetNextMerge();
-						if (merge != null)
-						{
-							writer.MergeInit(merge);
-							Enclosing_Instance.Message("  merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
-						}
-						else
-							break;
-					}
-					
-					Enclosing_Instance.Message("  merge thread: done");
-				}
-				catch (System.Exception exc)
-				{
-					
-					if (merge != null)
-					{
-						merge.SetException(exc);
-						writer.AddMergeException(merge);
-					}
-					
-					// Ignore the exception if it was due to abort:
-					if (!(exc is MergePolicy.MergeAbortedException))
-					{
-						lock (Enclosing_Instance)
-						{
-							Enclosing_Instance.exceptions.Add(exc);
-						}
-						
-						if (!Enclosing_Instance.suppressExceptions)
-						{
-							// suppressExceptions is normally only set during
-							// testing.
-							Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
-							throw new MergePolicy.MergeException(exc);
-						}
-					}
-				}
-				finally
-				{
-					lock (Enclosing_Instance)
-					{
-						Enclosing_Instance.mergeThreads.Remove(this);
-						System.Threading.Monitor.PulseAll(Enclosing_Instance);
-					}
-				}
-			}
-			
-			public override System.String ToString()
-			{
-				MergePolicy.OneMerge merge = GetRunningMerge();
-				if (merge == null)
-					merge = startMerge;
-				return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
-			}
-		}
-		
-		internal static bool anyExceptions = false;
-		
-		/// <summary>Used for testing </summary>
-		public static bool AnyUnhandledExceptions()
-		{
-			lock (allInstances.SyncRoot)
-			{
-				int count = allInstances.Count;
-				// Make sure all outstanding threads are done so we see
-				// any exceptions they may produce:
-				for (int i = 0; i < count; i++)
-					((ConcurrentMergeScheduler) allInstances[i]).Sync();
-				return anyExceptions;
-			}
-		}
-		
-		/// <summary>Used for testing </summary>
-		private void  AddMyself()
-		{
-			lock (allInstances.SyncRoot)
-			{
-				int size = 0;
-				int upto = 0;
-				for (int i = 0; i < size; i++)
-				{
-					ConcurrentMergeScheduler other = (ConcurrentMergeScheduler) allInstances[i];
-					if (!(other.closed && 0 == other.MergeThreadCount()))
-					// Keep this one for now: it still has threads or
-					// may spawn new threads
-						allInstances[upto++] = other;
-				}
-				((System.Collections.IList) ((System.Collections.ArrayList) allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
-				allInstances.Add(this);
-			}
-		}
-		
-		private bool suppressExceptions;
-		
-		/// <summary>Used for testing </summary>
-		internal virtual void  SetSuppressExceptions()
-		{
-			suppressExceptions = true;
-		}
-		
-		/// <summary>Used for testing </summary>
-		internal virtual void  ClearSuppressExceptions()
-		{
-			suppressExceptions = false;
-		}
-		
-		/// <summary>Used for testing </summary>
-		private static System.Collections.IList allInstances;
-		public static void  SetTestMode()
-		{
-			allInstances = new System.Collections.ArrayList();
-		}
+    /// <summary>
+    /// A {@link MergeScheduler} that runs each merge using a
+    /// separate thread, up until a maximum number of threads
+    /// ({@link #setMaxThreadCount}) at which when a merge is
+    /// needed, the thread(s) that are updating the index will
+    /// pause until one or more merges completes.  This is a
+    /// simple way to use concurrency in the indexing process
+    /// without having to create and manage application level
+    /// threads.
+    /// </summary>
+    public class ConcurrentMergeScheduler : MergeScheduler
+    {
+
+        private int mergeThreadPriority = -1;
+
+        protected System.Collections.Generic.List<MergeThread> mergeThreads = new System.Collections.Generic.List<MergeThread>();
+
+        // max number of threads allowed to be merging at once
+        private int maxThreadCount = 3;
+
+        private System.Collections.Generic.List<System.Exception> exceptions = new System.Collections.Generic.List<System.Exception>();
+        protected Directory dir;
+
+        private bool closed;
+        protected IndexWriter writer;
+        protected int mergeThreadCount;
+
+        public ConcurrentMergeScheduler()
+        {
+            if (allInstances != null)
+            {
+                // Only for testing
+                AddMyself();
+            }
+        }
+
+        /// <summary>Sets the max # simultaneous threads that may be
+        /// running.  If a merge is necessary yet we already have
+        /// this many threads running, the incoming thread (that
+        /// is calling add/updateDocument) will block until
+        /// a merge thread has completed.
+        /// </summary>
+        public virtual void SetMaxThreadCount(int count)
+        {
+            if (count < 1)
+                throw new System.ArgumentException("count should be at least 1");
+            maxThreadCount = count;
+        }
+
+        /// <summary>Get the max # simultaneous threads that may be</summary>
+        /// <seealso cref="setMaxThreadCount.">
+        /// </seealso>
+        public virtual int GetMaxThreadCount()
+        {
+            return maxThreadCount;
+        }
+
+        /// <summary>Return the priority that merge threads run at.  By
+        /// default the priority is 1 plus the priority of (ie,
+        /// slightly higher priority than) the first thread that
+        /// calls merge. 
+        /// </summary>
+        public virtual int GetMergeThreadPriority()
+        {
+            lock (this)
+            {
+                InitMergeThreadPriority();
+                return mergeThreadPriority;
+            }
+        }
+
+        /// <summary>Return the priority that merge threads run at. </summary>
+        public virtual void SetMergeThreadPriority(int pri)
+        {
+            lock (this)
+            {
+                if (pri > (int)System.Threading.ThreadPriority.Highest || pri < (int)System.Threading.ThreadPriority.Lowest)
+                    throw new System.ArgumentException("priority must be in range " + (int)System.Threading.ThreadPriority.Lowest + " .. " + (int)System.Threading.ThreadPriority.Highest + " inclusive");
+                mergeThreadPriority = pri;
+
+                int numThreads = MergeThreadCount();
+                for (int i = 0; i < numThreads; i++)
+                {
+                    MergeThread merge = mergeThreads[i];
+                    merge.SetThreadPriority(pri);
+                }
+            }
+        }
+
+        private void Message(System.String message)
+        {
+            if (writer != null)
+                writer.Message("CMS: " + message);
+        }
+
+        private void InitMergeThreadPriority()
+        {
+            lock (this)
+            {
+                if (mergeThreadPriority == -1)
+                {
+                    // Default to slightly higher priority than our calling thread
+                    mergeThreadPriority = 1 + (int)System.Threading.Thread.CurrentThread.Priority;
+                    if (mergeThreadPriority > (int)System.Threading.ThreadPriority.Highest)
+                        mergeThreadPriority = (int)System.Threading.ThreadPriority.Highest;
+                }
+            }
+        }
+
+        public override void Close()
+        {
+            closed = true;
+        }
+
+        public virtual void Sync()
+        {
+            lock (this)
+            {
+                while (MergeThreadCount() > 0)
+                {
+                    Message("now wait for threads; currently " + mergeThreads.Count + " still running");
+                    int count = mergeThreads.Count;
+                    for (int i = 0; i < count; i++)
+                        Message("    " + i + ": " + mergeThreads[i]);
+
+                    try
+                    {
+                        System.Threading.Monitor.Wait(this);
+                    }
+                    catch (System.Threading.ThreadInterruptedException)
+                    {
+                    }
+                }
+            }
+        }
+
+        private int MergeThreadCount()
+        {
+            lock (this)
+            {
+                int count = 0;
+                int numThreads = mergeThreads.Count;
+                for (int i = 0; i < numThreads; i++)
+                    if (mergeThreads[i].IsAlive)
+                        count++;
+                return count;
+            }
+        }
+
+        public override void Merge(IndexWriter writer)
+        {
+            this.writer = writer;
+            InitMergeThreadPriority();
+            dir = writer.GetDirectory();
+
+            // First, quickly run through the newly proposed merges
+            // and add any orthogonal merges (ie a merge not
+            // involving segments already pending to be merged) to
+            // the queue.  If we are way behind on merging, many of
+            // these newly proposed merges will likely already be
+            // registered.
+
+            Message("now merge");
+            Message("  index: " + writer.SegString());
+
+            // Iterate, pulling from the IndexWriter's queue of
+            // pending merges, until its empty:
+            while (true)
+            {
+                // TODO: we could be careful about which merges to do in
+                // the BG (eg maybe the "biggest" ones) vs FG, which
+                // merges to do first (the easiest ones?), etc.
+
+                MergePolicy.OneMerge merge = writer.GetNextMerge();
+                if (merge == null)
+                {
+                    Message("  no more merges pending; now return");
+                    return;
+                }
+
+                // We do this w/ the primary thread to keep
+                // deterministic assignment of segment names
+                writer.MergeInit(merge);
+
+                lock (this)
+                {
+                    while (MergeThreadCount() >= maxThreadCount)
+                    {
+                        Message("   too may merge threads running; stalling...");
+                        try
+                        {
+                            System.Threading.Monitor.Wait(this);
+                        }
+                        catch (System.Threading.ThreadInterruptedException)
+                        {
+                            SupportClass.ThreadClass.Current().Interrupt();
+                        }
+                    }
+
+                    Message("  consider merge " + merge.SegString(dir));
+
+                    System.Diagnostics.Debug.Assert(MergeThreadCount() < maxThreadCount);
+
+                    // OK to spawn a new merge thread to handle this
+                    // merge:
+                    MergeThread merger = GetMergeThread(writer, merge);
+                    mergeThreads.Add(merger);
+                    Message("    launch new thread [" + merger.Name + "]");
+                    merger.Start();
+                }
+            }
+        }
+
+        /// <summary>
+        /// Does the acural merge, by calling IndexWriter.Merge().
+        /// </summary>
+        /// <param name="merge"></param>
+        virtual protected void DoMerge(MergePolicy.OneMerge merge)
+        {
+            writer.Merge(merge);
+        }
+
+        /// <summary>
+        /// Create and return a new MergeThread.
+        /// </summary>
+        /// <param name="writer"></param>
+        /// <param name="merge"></param>
+        /// <returns></returns>
+        virtual protected MergeThread GetMergeThread(IndexWriter writer, MergePolicy.OneMerge merge)
+        {
+            MergeThread thread = new MergeThread(this, writer, merge);
+            thread.SetThreadPriority(mergeThreadPriority);
+            thread.IsBackground = true;
+            thread.Name = "Lucene Merge Thread #" + mergeThreadCount++;
+            return thread;
+        }
+
+        protected class MergeThread : SupportClass.ThreadClass
+        {
+            private void InitBlock(ConcurrentMergeScheduler enclosingInstance)
+            {
+                this.enclosingInstance = enclosingInstance;
+            }
+            private ConcurrentMergeScheduler enclosingInstance;
+            public ConcurrentMergeScheduler Enclosing_Instance
+            {
+                get
+                {
+                    return enclosingInstance;
+                }
+
+            }
+
+            internal IndexWriter writer;
+            internal MergePolicy.OneMerge startMerge;
+            internal MergePolicy.OneMerge runningMerge;
+
+            public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
+            {
+                InitBlock(enclosingInstance);
+                this.writer = writer;
+                this.startMerge = startMerge;
+            }
+
+            public virtual void SetRunningMerge(MergePolicy.OneMerge merge)
+            {
+                lock (this)
+                {
+                    runningMerge = merge;
+                }
+            }
+
+            public virtual MergePolicy.OneMerge GetRunningMerge()
+            {
+                lock (this)
+                {
+                    return runningMerge;
+                }
+            }
+
+            public virtual void SetThreadPriority(int pri)
+            {
+                try
+                {
+                    Priority = (System.Threading.ThreadPriority)pri;
+                }
+                catch (System.NullReferenceException)
+                {
+                    // Strangely, Sun's JDK 1.5 on Linux sometimes
+                    // throws NPE out of here...
+                }
+                catch (System.Security.SecurityException)
+                {
+                    // Ignore this because we will still run fine with
+                    // normal thread priority
+                }
+            }
+
+            override public void Run()
+            {
+
+                // First time through the while loop we do the merge
+                // that we were started with:
+                MergePolicy.OneMerge merge = this.startMerge;
+
+                try
+                {
+
+                    Enclosing_Instance.Message("  merge thread: start");
+
+                    while (true)
+                    {
+                        SetRunningMerge(merge);
+                        Enclosing_Instance.DoMerge(merge);
+
+                        // Subsequent times through the loop we do any new
+                        // merge that writer says is necessary:
+                        merge = writer.GetNextMerge();
+                        if (merge != null)
+                        {
+                            writer.MergeInit(merge);
+                            Enclosing_Instance.Message("  merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
+                        }
+                        else
+                            break;
+                    }
+
+                    Enclosing_Instance.Message("  merge thread: done");
+                }
+                catch (System.Exception exc)
+                {
+                    // Ignore the exception if it was due to abort:
+                    if (!(exc is MergePolicy.MergeAbortedException))
+                    {
+                        lock (Enclosing_Instance)
+                        {
+                            Enclosing_Instance.exceptions.Add(exc);
+                        }
+
+                        if (!Enclosing_Instance.suppressExceptions)
+                        {
+                            // suppressExceptions is normally only set during
+                            // testing.
+                            Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
+                            Enclosing_Instance.HandleMergeException(exc);
+                        }
+                    }
+                }
+                finally
+                {
+                    lock (Enclosing_Instance)
+                    {
+                        System.Threading.Monitor.PulseAll(Enclosing_Instance);
+                        bool removed = Enclosing_Instance.mergeThreads.Remove(this);
+                        System.Diagnostics.Debug.Assert(removed);
+                    }
+                }
+            }
+
+            public override System.String ToString()
+            {
+                MergePolicy.OneMerge merge = GetRunningMerge();
+                if (merge == null)
+                    merge = startMerge;
+                return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
+            }
+        }
+
+        virtual protected void HandleMergeException(System.Exception exc)
+        {
+            throw new MergePolicy.MergeException(exc, dir);
+        }
+
+        internal static bool anyExceptions = false;
+
+        /// <summary>Used for testing </summary>
+        public static bool AnyUnhandledExceptions()
+        {
+            lock (allInstances.SyncRoot)
+            {
+                int count = allInstances.Count;
+                // Make sure all outstanding threads are done so we see
+                // any exceptions they may produce:
+                for (int i = 0; i < count; i++)
+                    ((ConcurrentMergeScheduler)allInstances[i]).Sync();
+                bool v = anyExceptions;
+                anyExceptions = false;
+                return v;
+            }
+        }
+
+        public static void ClearUnhandledExceptions()
+        {
+            lock (allInstances)
+            {
+                anyExceptions = false;
+            }
+        }
+
+        /// <summary>Used for testing </summary>
+        private void AddMyself()
+        {
+            lock (allInstances.SyncRoot)
+            {
+                int size = 0;
+                int upto = 0;
+                for (int i = 0; i < size; i++)
+                {
+                    ConcurrentMergeScheduler other = (ConcurrentMergeScheduler)allInstances[i];
+                    if (!(other.closed && 0 == other.MergeThreadCount()))
+                        // Keep this one for now: it still has threads or
+                        // may spawn new threads
+                        allInstances[upto++] = other;
+                }
+                ((System.Collections.IList)((System.Collections.ArrayList)allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
+                allInstances.Add(this);
+            }
+        }
+
+        private bool suppressExceptions;
+
+        /// <summary>Used for testing </summary>
+        internal virtual void SetSuppressExceptions()
+        {
+            suppressExceptions = true;
+        }
+
+        /// <summary>Used for testing </summary>
+        internal virtual void ClearSuppressExceptions()
+        {
+            suppressExceptions = false;
+        }
+
+        /// <summary>Used for testing </summary>
+        private static System.Collections.IList allInstances;
+        public static void SetTestMode()
+        {
+            allInstances = new System.Collections.ArrayList();
+        }
 
         public void SetSuppressExceptions_ForNUnitTest()
         {
             SetSuppressExceptions();
         }
-        
+
         public void ClearSuppressExceptions_ForNUnitTest()
         {
             ClearSuppressExceptions();
         }
-	}
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListWriter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs Wed Jul 29 18:04:12 2009
@@ -61,16 +61,18 @@
 			this.curStorePayloads = storePayloads;
 			this.curPayloadLength = payloadLength;
 			this.curFreqPointer = freqOutput.GetFilePointer();
-			this.curProxPointer = proxOutput.GetFilePointer();
+            if (proxOutput != null)
+			    this.curProxPointer = proxOutput.GetFilePointer();
 		}
 		
 		protected internal override void  ResetSkip()
 		{
 			base.ResetSkip();
-			for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0;
-			for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list
-			for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.GetFilePointer();
-			for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.GetFilePointer();
+			SupportClass.CollectionsSupport.ArrayFill(lastSkipDoc, 0);
+			SupportClass.CollectionsSupport.ArrayFill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list
+			SupportClass.CollectionsSupport.ArrayFill(lastSkipFreqPointer, freqOutput.GetFilePointer());
+            if (proxOutput != null)
+			    SupportClass.CollectionsSupport.ArrayFill(lastSkipProxPointer, proxOutput.GetFilePointer());
 		}
 		
 		protected internal override void  WriteSkipData(int level, IndexOutput skipBuffer)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DirectoryIndexReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs Wed Jul 29 18:04:12 2009
@@ -16,6 +16,7 @@
  */
 
 using System;
+using System.Collections.Generic;
 
 using Directory = Lucene.Net.Store.Directory;
 using Lock = Lucene.Net.Store.Lock;
@@ -23,357 +24,529 @@
 
 namespace Lucene.Net.Index
 {
-	
-	/// <summary> IndexReader implementation that has access to a Directory. 
-	/// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
-	/// "own" the directory, which means that they try to acquire a write lock
-	/// whenever index modifications are performed.
-	/// </summary>
-	abstract public class DirectoryIndexReader : IndexReader
-	{
-		private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
-		{
-			private void  InitBlock(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy)
-			{
-				this.closeDirectory = closeDirectory;
-				this.deletionPolicy = deletionPolicy;
-			}
-			private bool closeDirectory;
-			private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
-			internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1) : base(Param1)
-			{
-				InitBlock(closeDirectory, deletionPolicy);
-			}
-			
-			protected internal override System.Object DoBody(System.String segmentFileName)
-			{
-				
-				SegmentInfos infos = new SegmentInfos();
-				infos.Read(directory, segmentFileName);
-				
-				DirectoryIndexReader reader;
-				
-				if (infos.Count == 1)
-				{
-					// index is optimized
-					reader = SegmentReader.Get(infos, infos.Info(0), closeDirectory);
-				}
-				else
-				{
-					reader = new MultiSegmentReader(directory, infos, closeDirectory);
-				}
-				reader.SetDeletionPolicy(deletionPolicy);
-				return reader;
-			}
-		}
-		
-		private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
-		{
-			private void  InitBlock(DirectoryIndexReader enclosingInstance)
-			{
-				this.enclosingInstance = enclosingInstance;
-			}
-			private DirectoryIndexReader enclosingInstance;
-			public DirectoryIndexReader Enclosing_Instance
-			{
-				get
-				{
-					return enclosingInstance;
-				}
-				
-			}
-			internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1) : base(Param1)
-			{
-				InitBlock(enclosingInstance);
-			}
-			
-			protected internal override System.Object DoBody(System.String segmentFileName)
-			{
-				SegmentInfos infos = new SegmentInfos();
-				infos.Read(directory, segmentFileName);
-				
-				DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
-				
-				if (Enclosing_Instance != newReader)
-				{
-					newReader.Init(directory, infos, Enclosing_Instance.closeDirectory);
-					newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
-				}
-				
-				return newReader;
-			}
-		}
-		protected internal Directory directory;
-		protected internal bool closeDirectory;
-		private IndexDeletionPolicy deletionPolicy;
-		
-		private SegmentInfos segmentInfos;
-		private Lock writeLock;
-		private bool stale;
-		
-		/// <summary>Used by commit() to record pre-commit state in case
-		/// rollback is necessary 
-		/// </summary>
-		private bool rollbackHasChanges;
-		private SegmentInfos rollbackSegmentInfos;
-		
-		
-		internal virtual void  Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory)
-		{
-			this.directory = directory;
-			this.segmentInfos = segmentInfos;
-			this.closeDirectory = closeDirectory;
-		}
-		
-		protected internal DirectoryIndexReader()
-		{
-		}
-		
-		internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory) : base()
-		{
-			Init(directory, segmentInfos, closeDirectory);
-		}
-		
-		internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
-		{
-			
-			return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory).Run();
-		}
-		
-		
-		public override IndexReader Reopen()
-		{
-			lock (this)
-			{
-				EnsureOpen();
-				
-				if (this.hasChanges || this.IsCurrent())
-				{
-					// the index hasn't changed - nothing to do here
-					return this;
-				}
-				
-				return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile1(this, directory).Run();
-			}
-		}
-		
-		/// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
-		protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
-		
-		public virtual void  SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
-		{
-			this.deletionPolicy = deletionPolicy;
-		}
-		
-		/// <summary>Returns the directory this index resides in.</summary>
-		public override Directory Directory()
-		{
-			EnsureOpen();
-			return directory;
-		}
-		
-		/// <summary> Version number when this IndexReader was opened.</summary>
-		public override long GetVersion()
-		{
-			EnsureOpen();
-			return segmentInfos.GetVersion();
-		}
-		
-		/// <summary> Check whether this IndexReader is still using the
-		/// current (i.e., most recently committed) version of the
-		/// index.  If a writer has committed any changes to the
-		/// index since this reader was opened, this will return
-		/// <code>false</code>, in which case you must open a new
-		/// IndexReader in order to see the changes.  See the
-		/// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
-		/// flag which controls when the {@link IndexWriter}
-		/// actually commits changes to the index.
-		/// 
-		/// </summary>
-		/// <throws>  CorruptIndexException if the index is corrupt </throws>
-		/// <throws>  IOException if there is a low-level IO error </throws>
-		public override bool IsCurrent()
-		{
-			EnsureOpen();
-			return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
-		}
-		
-		/// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
-		/// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
-		/// </returns>
-		public override bool IsOptimized()
-		{
-			EnsureOpen();
-			return segmentInfos.Count == 1 && HasDeletions() == false;
-		}
-		
-		protected internal override void  DoClose()
-		{
-			if (closeDirectory)
-				directory.Close();
-		}
-		
-		/// <summary> Commit changes resulting from delete, undeleteAll, or
-		/// setNorm operations
-		/// 
-		/// If an exception is hit, then either no changes or all
-		/// changes will have been committed to the index
-		/// (transactional semantics).
-		/// </summary>
-		/// <throws>  IOException if there is a low-level IO error </throws>
-		protected internal override void  DoCommit()
-		{
-			if (hasChanges)
-			{
-				if (segmentInfos != null)
-				{
-					
-					// Default deleter (for backwards compatibility) is
-					// KeepOnlyLastCommitDeleter:
-					IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
-					
-					// Checkpoint the state we are about to change, in
-					// case we have to roll back:
-					StartCommit();
-					
-					bool success = false;
-					try
-					{
-						CommitChanges();
-						segmentInfos.Write(directory);
-						success = true;
-					}
-					finally
-					{
-						
-						if (!success)
-						{
-							
-							// Rollback changes that were made to
-							// SegmentInfos but failed to get [fully]
-							// committed.  This way this reader instance
-							// remains consistent (matched to what's
-							// actually in the index):
-							RollbackCommit();
-							
-							// Recompute deletable files & remove them (so
-							// partially written .del files, etc, are
-							// removed):
-							deleter.Refresh();
-						}
-					}
-					
-					// Have the deleter remove any now unreferenced
-					// files due to this commit:
-					deleter.Checkpoint(segmentInfos, true);
-					
-					if (writeLock != null)
-					{
-						writeLock.Release(); // release write lock
-						writeLock = null;
-					}
-				}
-				else
-					CommitChanges();
-			}
-			hasChanges = false;
-		}
-		
-		protected internal abstract void  CommitChanges();
-		
-		/// <summary> Tries to acquire the WriteLock on this directory.
-		/// this method is only valid if this IndexReader is directory owner.
-		/// 
-		/// </summary>
-		/// <throws>  StaleReaderException if the index has changed </throws>
-		/// <summary> since this reader was opened
-		/// </summary>
-		/// <throws>  CorruptIndexException if the index is corrupt </throws>
-		/// <throws>  LockObtainFailedException if another writer </throws>
-		/// <summary>  has this index open (<code>write.lock</code> could not
-		/// be obtained)
-		/// </summary>
-		/// <throws>  IOException if there is a low-level IO error </throws>
-		protected internal override void  AcquireWriteLock()
-		{
-			if (segmentInfos != null)
-			{
-				EnsureOpen();
-				if (stale)
-					throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
-				
-				if (this.writeLock == null)
-				{
-					Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
-					if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
-					// obtain write lock
-					{
-						throw new LockObtainFailedException("Index locked for write: " + writeLock);
-					}
-					this.writeLock = writeLock;
-					
-					// we have to check whether index has changed since this reader was opened.
-					// if so, this reader is no longer valid for deletion
-					if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
-					{
-						stale = true;
-						this.writeLock.Release();
-						this.writeLock = null;
-						throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
-					}
-				}
-			}
-		}
-		
-		/// <summary> Should internally checkpoint state that will change
-		/// during commit so that we can rollback if necessary.
-		/// </summary>
-		internal virtual void  StartCommit()
-		{
-			if (segmentInfos != null)
-			{
-				rollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
-			}
-			rollbackHasChanges = hasChanges;
-		}
-		
-		/// <summary> Rolls back state to just before the commit (this is
-		/// called by commit() if there is some exception while
-		/// committing).
-		/// </summary>
-		internal virtual void  RollbackCommit()
-		{
-			if (segmentInfos != null)
-			{
-				for (int i = 0; i < segmentInfos.Count; i++)
-				{
-					// Rollback each segmentInfo.  Because the
-					// SegmentReader holds a reference to the
-					// SegmentInfo we can't [easily] just replace
-					// segmentInfos, so we reset it in place instead:
-					segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
-				}
-				rollbackSegmentInfos = null;
-			}
-			
-			hasChanges = rollbackHasChanges;
-		}
-		
-		/// <summary>Release the write lock, if needed. </summary>
-		~DirectoryIndexReader()
-		{
-			try
-			{
-				if (writeLock != null)
-				{
-					writeLock.Release(); // release write lock
-					writeLock = null;
-				}
-			}
-			finally
+
+    /// <summary> IndexReader implementation that has access to a Directory. 
+    /// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
+    /// "own" the directory, which means that they try to acquire a write lock
+    /// whenever index modifications are performed.
+    /// </summary>
+    abstract public class DirectoryIndexReader : IndexReader
+    {
+        protected internal Directory directory;
+        protected internal bool closeDirectory;
+        private IndexDeletionPolicy deletionPolicy;
+
+        private SegmentInfos segmentInfos;
+        private Lock writeLock;
+        private bool stale;
+        private readonly IDictionary<string, string> synced = new Dictionary<string, string>();
+
+        /// <summary>Used by commit() to record pre-commit state in case
+        /// rollback is necessary 
+        /// </summary>
+        private bool rollbackHasChanges;
+        private SegmentInfos rollbackSegmentInfos;
+
+        protected internal bool readOnly;
+
+        internal virtual void Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory, bool readOnly)
+        {
+            this.directory = directory;
+            this.segmentInfos = segmentInfos;
+            this.closeDirectory = closeDirectory;
+            this.readOnly = readOnly;
+
+            if (!readOnly && segmentInfos != null)
+            {
+                // we assume that this segments_N was properly sync'd prior
+                for (int i = 0; i < segmentInfos.Count; i++)
+                {
+                    SegmentInfo info = segmentInfos.Info(i);
+                    IList<string> files = info.Files();
+                    for (int j = 0; j < files.Count; j++)
+                        synced[files[j]] = files[j];
+                }
+            }
+        }
+
+        protected internal DirectoryIndexReader()
+        {
+        }
+
+        internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory, bool readOnly)
+            : base()
+        {
+            Init(directory, segmentInfos, closeDirectory, readOnly);
+        }
+
+        internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
+        {
+            return Open(directory, closeDirectory, deletionPolicy, null, false);
+        }
+
+        internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy, IndexCommit commit, bool readOnly)
+        {
+            SegmentInfos.FindSegmentsFile finder = new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory, readOnly);
+
+            if (commit == null)
+                return (DirectoryIndexReader) finder.Run();
+            else
+            {
+                if (directory != commit.GetDirectory())
+                    throw new System.IO.IOException("the specified commit does not match the specified Directory");
+                // this can and will directly throw IOException if the specified commit point has been deleted
+                return (DirectoryIndexReader)finder.DoBody(commit.GetSegmentsFileName());
+            }
+        }
+
+        private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
+        {
+            private bool closeDirectory;
+            private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
+            private bool readOnly;
+
+            internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1, bool readOnly)
+                : base(Param1)
+            {
+                this.closeDirectory = closeDirectory;
+                this.deletionPolicy = deletionPolicy;
+                this.readOnly = readOnly;
+            }
+
+            protected internal override object DoBody(System.String segmentFileName)
+            {
+                SegmentInfos infos = new SegmentInfos();
+                infos.Read(directory, segmentFileName);
+
+                DirectoryIndexReader reader;
+                if (infos.Count == 1)
+                {
+                    // index is optimized
+                    reader = SegmentReader.Get(readOnly, infos, infos.Info(0), closeDirectory);
+                }
+                else if (readOnly)
+                {
+                    reader = new ReadOnlyMultiSegmentReader(directory, infos, closeDirectory);
+                }
+                else
+                {
+                    reader = new MultiSegmentReader(directory, infos, closeDirectory, false);
+                }
+                reader.SetDeletionPolicy(deletionPolicy);
+
+                return reader;
+            }
+        }
+
+        public override IndexReader Reopen()
+        {
+            lock (this)
+            {
+                EnsureOpen();
+
+                if (this.hasChanges || this.IsCurrent())
+                {
+                    // this has changes, therefore we have the lock and don't need to reopen
+                    // OR: the index in the directory hasn't changed - nothing to do here
+                    return this;
+                }
+
+                return (DirectoryIndexReader)new AnonymousClassFindSegmentsFile1(this, directory).Run();
+            }
+        }
+
+        private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
+        {
+            private DirectoryIndexReader enclosingInstance;
+
+            public DirectoryIndexReader Enclosing_Instance
+            {
+                get
+                {
+                    return enclosingInstance;
+                }
+
+            }
+
+            internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1)
+                : base(Param1)
             {
-				// {{Aroush-2.3.1}} do we need to call Finalize() here?
+                this.enclosingInstance = enclosingInstance;
             }
-		}
-	}
+
+            protected internal override object DoBody(System.String segmentFileName)
+            {
+                SegmentInfos infos = new SegmentInfos();
+                infos.Read(directory, segmentFileName);
+
+                DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
+
+                if (Enclosing_Instance != newReader)
+                {
+                    newReader.Init(directory, infos, Enclosing_Instance.closeDirectory, Enclosing_Instance.readOnly);
+                    newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
+                }
+
+                return newReader;
+            }
+        }
+
+        /// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
+        protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
+
+        public virtual void SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
+        {
+            this.deletionPolicy = deletionPolicy;
+        }
+
+        /// <summary>Returns the directory this index resides in.</summary>
+        public override Directory Directory()
+        {
+            EnsureOpen();
+            return directory;
+        }
+
+        /// <summary> Version number when this IndexReader was opened.</summary>
+        public override long GetVersion()
+        {
+            EnsureOpen();
+            return segmentInfos.GetVersion();
+        }
+
+        /// <summary> Check whether this IndexReader is still using the
+        /// current (i.e., most recently committed) version of the
+        /// index.  If a writer has committed any changes to the
+        /// index since this reader was opened, this will return
+        /// <code>false</code>, in which case you must open a new
+        /// IndexReader in order to see the changes.  See the
+        /// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
+        /// flag which controls when the {@link IndexWriter}
+        /// actually commits changes to the index.
+        /// 
+        /// </summary>
+        /// <throws>  CorruptIndexException if the index is corrupt </throws>
+        /// <throws>  IOException if there is a low-level IO error </throws>
+        public override bool IsCurrent()
+        {
+            EnsureOpen();
+            return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
+        }
+
+        /// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
+        /// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
+        /// </returns>
+        public override bool IsOptimized()
+        {
+            EnsureOpen();
+            return segmentInfos.Count == 1 && HasDeletions() == false;
+        }
+
+        protected internal override void DoClose()
+        {
+            if (closeDirectory)
+                directory.Close();
+        }
+
+        /// <summary> Commit changes resulting from delete, undeleteAll, or
+        /// setNorm operations
+        /// 
+        /// If an exception is hit, then either no changes or all
+        /// changes will have been committed to the index
+        /// (transactional semantics).
+        /// </summary>
+        /// <throws>  IOException if there is a low-level IO error </throws>
+        protected internal override void DoCommit()
+        {
+            if (hasChanges)
+            {
+                if (segmentInfos != null)
+                {
+
+                    // Default deleter (for backwards compatibility) is
+                    // KeepOnlyLastCommitDeleter:
+                    IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
+
+                    // Checkpoint the state we are about to change, in
+                    // case we have to roll back:
+                    StartCommit();
+
+                    bool success = false;
+                    try
+                    {
+                        CommitChanges();
+
+                        // sync all the files we just wrote
+                        for (int i = 0; i < segmentInfos.Count; i++)
+                        {
+                            SegmentInfo info = segmentInfos.Info(i);
+                            IList<string> files = info.Files();
+                            for (int j = 0; j < files.Count; j++)
+                            {
+                                string fileName = files[j];
+                                if (!synced.ContainsKey(fileName))
+                                {
+                                    System.Diagnostics.Debug.Assert(directory.FileExists(fileName));
+                                    directory.Sync(fileName);
+                                    synced[fileName] = fileName;
+                                }
+                            }
+                        }
+
+                        segmentInfos.Commit(directory);
+                        success = true;
+                    }
+                    finally
+                    {
+
+                        if (!success)
+                        {
+
+                            // Rollback changes that were made to
+                            // SegmentInfos but failed to get [fully]
+                            // committed.  This way this reader instance
+                            // remains consistent (matched to what's
+                            // actually in the index):
+                            RollbackCommit();
+
+                            // Recompute deletable files & remove them (so
+                            // partially written .del files, etc, are
+                            // removed):
+                            deleter.Refresh();
+                        }
+                    }
+
+                    // Have the deleter remove any now unreferenced
+                    // files due to this commit:
+                    deleter.Checkpoint(segmentInfos, true);
+
+                    if (writeLock != null)
+                    {
+                        writeLock.Release(); // release write lock
+                        writeLock = null;
+                    }
+                }
+                else
+                    CommitChanges();
+            }
+            hasChanges = false;
+        }
+
+        protected internal abstract void CommitChanges();
+
+        /// <summary> Tries to acquire the WriteLock on this directory.
+        /// this method is only valid if this IndexReader is directory owner.
+        /// 
+        /// </summary>
+        /// <throws>  StaleReaderException if the index has changed </throws>
+        /// <summary> since this reader was opened
+        /// </summary>
+        /// <throws>  CorruptIndexException if the index is corrupt </throws>
+        /// <throws>  LockObtainFailedException if another writer </throws>
+        /// <summary>  has this index open (<code>write.lock</code> could not
+        /// be obtained)
+        /// </summary>
+        /// <throws>  IOException if there is a low-level IO error </throws>
+        protected internal override void AcquireWriteLock()
+        {
+            if (segmentInfos != null)
+            {
+                EnsureOpen();
+                if (stale)
+                    throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+
+                if (this.writeLock == null)
+                {
+                    Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
+                    if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
+                    // obtain write lock
+                    {
+                        throw new LockObtainFailedException("Index locked for write: " + writeLock);
+                    }
+                    this.writeLock = writeLock;
+
+                    // we have to check whether index has changed since this reader was opened.
+                    // if so, this reader is no longer valid for deletion
+                    if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
+                    {
+                        stale = true;
+                        this.writeLock.Release();
+                        this.writeLock = null;
+                        throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+                    }
+                }
+            }
+        }
+
+        /// <summary> Should internally checkpoint state that will change
+        /// during commit so that we can rollback if necessary.
+        /// </summary>
+        internal virtual void StartCommit()
+        {
+            if (segmentInfos != null)
+            {
+                rollbackSegmentInfos = (SegmentInfos)segmentInfos.Clone();
+            }
+            rollbackHasChanges = hasChanges;
+        }
+
+        /// <summary> Rolls back state to just before the commit (this is
+        /// called by commit() if there is some exception while
+        /// committing).
+        /// </summary>
+        internal virtual void RollbackCommit()
+        {
+            if (segmentInfos != null)
+            {
+                for (int i = 0; i < segmentInfos.Count; i++)
+                {
+                    // Rollback each segmentInfo.  Because the
+                    // SegmentReader holds a reference to the
+                    // SegmentInfo we can't [easily] just replace
+                    // segmentInfos, so we reset it in place instead:
+                    segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
+                }
+                rollbackSegmentInfos = null;
+            }
+
+            hasChanges = rollbackHasChanges;
+        }
+
+        /// <summary>Release the write lock, if needed. </summary>
+        ~DirectoryIndexReader()
+        {
+            try
+            {
+                if (writeLock != null)
+                {
+                    writeLock.Release(); // release write lock
+                    writeLock = null;
+                }
+            }
+            finally
+            {
+                // {{Aroush-2.3.1}} do we need to call Finalize() here?
+            }
+        }
+
+        private class ReaderCommit : IndexCommit
+        {
+            private string segmentsFileName;
+            internal ICollection<string> files;
+            internal Directory dir;
+            internal long generation;
+            internal long version;
+            internal readonly bool isOptimized;
+
+            internal ReaderCommit(SegmentInfos infos, Directory dir)
+            {
+                segmentsFileName = infos.GetCurrentSegmentFileName();
+                this.dir = dir;
+                int size = infos.Count;
+                files = new List<string>(size);
+                files.Add(segmentsFileName);
+                for (int i = 0; i < size; i++)
+                {
+                    SegmentInfo info = infos.Info(i);
+                    if (info.dir == dir)
+                        SupportClass.CollectionsSupport.AddAll(info.Files(), files);
+                }
+                version = infos.GetVersion();
+                generation = infos.GetGeneration();
+                isOptimized = infos.Count == 1 && !infos.Info(0).HasDeletions();
+            }
+
+            public override bool IsOptimized()
+            {
+                return isOptimized;
+            }
+            public override string GetSegmentsFileName()
+            {
+                return segmentsFileName;
+            }
+            public override ICollection<string> GetFileNames()
+            {
+                return files;
+            }
+            public override Directory GetDirectory()
+            {
+                return dir;
+            }
+            public override long GetVersion()
+            {
+                return version;
+            }
+            public override long GetGeneration()
+            {
+                return generation;
+            }
+            public override bool IsDeleted()
+            {
+                return false;
+            }
+        }
+
+        /**
+         * Expert: return the IndexCommit that this reader has
+         * opened.
+         *
+         * <p><b>WARNING</b>: this API is new and experimental and
+         * may suddenly change.</p>
+         */
+        public override IndexCommit GetIndexCommit()
+        {
+            return new ReaderCommit(segmentInfos, directory);
+        }
+
+        /** @see IndexReader#listCommits */
+        public new static ICollection<IndexCommitPoint> ListCommits(Directory dir)
+        {
+
+            string[] files = dir.List();
+            if (files == null)
+                throw new System.IO.IOException("cannot read directory " + dir + ": list() returned null");
+
+            ICollection<IndexCommitPoint> commits = new List<IndexCommitPoint>();
+
+            SegmentInfos latest = new SegmentInfos();
+            latest.Read(dir);
+            long currentGen = latest.GetGeneration();
+
+            commits.Add(new ReaderCommit(latest, dir));
+
+            for (int i = 0; i < files.Length; i++)
+            {
+
+                String fileName = files[i];
+
+                if (fileName.StartsWith(IndexFileNames.SEGMENTS) &&
+                    !fileName.Equals(IndexFileNames.SEGMENTS_GEN) &&
+                    SegmentInfos.GenerationFromSegmentsFileName(fileName) < currentGen)
+                {
+                    SegmentInfos sis = new SegmentInfos();
+                    try
+                    {
+                        // IOException allowed to throw there, in case
+                        // segments_N is corrupt
+                        sis.Read(dir, fileName);
+                    }
+                    catch (System.Exception)
+                    {
+                        // LUCENE-948: on NFS (and maybe others), if
+                        // you have writers switching back and forth
+                        // between machines, it's very likely that the
+                        // dir listing will be stale and will claim a
+                        // file segments_X exists when in fact it
+                        // doesn't.  So, we catch this and handle it
+                        // as if the file does not exist
+                        sis = null;
+                    }
+
+                    if (sis != null)
+                        commits.Add(new ReaderCommit(sis, dir));
+                }
+            }
+
+            return commits;
+        }
+    }
 }
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocConsumer.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocConsumerPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumer.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumerPerField.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumers.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumersPerField.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Fieldable = Lucene.Net.Documents.Fieldable;
+
+namespace Lucene.Net.Index
+{
+    internal sealed class DocFieldConsumersPerField : DocFieldConsumerPerField
+    {
+
+        internal readonly DocFieldConsumerPerField one;
+        internal readonly DocFieldConsumerPerField two;
+        internal readonly DocFieldConsumersPerThread perThread;
+
+        public DocFieldConsumersPerField(DocFieldConsumersPerThread perThread, DocFieldConsumerPerField one, DocFieldConsumerPerField two)
+        {
+            this.perThread = perThread;
+            this.one = one;
+            this.two = two;
+        }
+
+        internal override void processFields(Fieldable[] fields, int count)
+        {
+            one.processFields(fields, count);
+            two.processFields(fields, count);
+        }
+
+        internal override void abort()
+        {
+            try
+            {
+                one.abort();
+            }
+            finally
+            {
+                two.abort();
+            }
+        }
+    }
+}

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessor.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessorPerField.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverter.cs?rev=798995&view=auto
==============================================================================
Binary file - no diff available.

Propchange: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverterPerField.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Fieldable = Lucene.Net.Documents.Fieldable;
+using Token = Lucene.Net.Analysis.Token;
+using TokenStream = Lucene.Net.Analysis.TokenStream;
+
+namespace Lucene.Net.Index
+{
+    /// <summary>
+    /// Holds state for inverting all occurrences of a single
+    /// field in the document.  This class doesn't do anything
+    /// itself; instead, it forwards the tokens produced by
+    /// analysis to its own consumer
+    /// (InvertedDocConsumerPerField).  It also interacts with an
+    /// endConsumer (InvertedDocEndConsumerPerField).
+    /// </summary>
+    internal sealed class DocInverterPerField : DocFieldConsumerPerField
+    {
+
+        private readonly DocInverterPerThread perThread;
+        private readonly FieldInfo fieldInfo;
+        internal readonly InvertedDocConsumerPerField consumer;
+        internal readonly InvertedDocEndConsumerPerField endConsumer;
+        internal readonly DocumentsWriter.DocState docState;
+        internal readonly DocInverter.FieldInvertState fieldState;
+
+        public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo)
+        {
+            this.perThread = perThread;
+            this.fieldInfo = fieldInfo;
+            docState = perThread.docState;
+            fieldState = perThread.fieldState;
+            this.consumer = perThread.consumer.addField(this, fieldInfo);
+            this.endConsumer = perThread.endConsumer.addField(this, fieldInfo);
+        }
+
+        internal override void abort()
+        {
+            consumer.abort();
+            endConsumer.abort();
+        }
+
+        internal override void processFields(Fieldable[] fields,
+                                  int count)
+        {
+
+            fieldState.reset(docState.doc.GetBoost());
+
+            int maxFieldLength = docState.maxFieldLength;
+
+            bool doInvert = consumer.start(fields, count);
+
+            for (int i = 0; i < count; i++)
+            {
+
+                Fieldable field = fields[i];
+
+                // TODO FI: this should be "genericized" to querying
+                // consumer if it wants to see this particular field
+                // tokenized.
+                if (field.IsIndexed() && doInvert)
+                {
+
+                    if (fieldState.length > 0)
+                        fieldState.position += docState.analyzer.GetPositionIncrementGap(fieldInfo.name);
+
+                    if (!field.IsTokenized())
+                    {		  // un-tokenized field
+                        string stringValue = field.StringValue();
+                        int valueLength = stringValue.Length;
+                        Token token = perThread.localToken.Reinit(stringValue, fieldState.offset, fieldState.offset + valueLength);
+                        bool success = false;
+                        try
+                        {
+                            consumer.add(token);
+                            success = true;
+                        }
+                        finally
+                        {
+                            if (!success)
+                                docState.docWriter.SetAborting();
+                        }
+                        fieldState.offset += valueLength;
+                        fieldState.length++;
+                        fieldState.position++;
+                    }
+                    else
+                    {                                  // tokenized field
+                        TokenStream stream;
+                        TokenStream streamValue = field.TokenStreamValue();
+
+                        if (streamValue != null)
+                            stream = streamValue;
+                        else
+                        {
+                            // the field does not have a TokenStream,
+                            // so we have to obtain one from the analyzer
+                            System.IO.TextReader reader;			  // find or make Reader
+                            System.IO.TextReader readerValue = field.ReaderValue();
+
+                            if (readerValue != null)
+                                reader = readerValue;
+                            else
+                            {
+                                string stringValue = field.StringValue();
+                                if (stringValue == null)
+                                    throw new System.ArgumentException("field must have either TokenStream, string or Reader value");
+                                perThread.stringReader.Init(stringValue);
+                                reader = perThread.stringReader;
+                            }
+
+                            // Tokenize field and add to postingTable
+                            stream = docState.analyzer.ReusableTokenStream(fieldInfo.name, reader);
+                        }
+
+                        // reset the TokenStream to the first token
+                        stream.Reset();
+
+                        try
+                        {
+                            int offsetEnd = fieldState.offset - 1;
+                            Token localToken = perThread.localToken;
+                            for (; ; )
+                            {
+
+                                // If we hit an exception in stream.next below
+                                // (which is fairly common, eg if analyzer
+                                // chokes on a given document), then it's
+                                // non-aborting and (above) this one document
+                                // will be marked as deleted, but still
+                                // consume a docID
+                                Token token = stream.Next(localToken);
+
+                                if (token == null) break;
+                                fieldState.position += (token.GetPositionIncrement() - 1);
+                                bool success = false;
+                                try
+                                {
+                                    // If we hit an exception in here, we abort
+                                    // all buffered documents since the last
+                                    // flush, on the likelihood that the
+                                    // internal state of the consumer is now
+                                    // corrupt and should not be flushed to a
+                                    // new segment:
+                                    consumer.add(token);
+                                    success = true;
+                                }
+                                finally
+                                {
+                                    if (!success)
+                                        docState.docWriter.SetAborting();
+                                }
+                                fieldState.position++;
+                                offsetEnd = fieldState.offset + token.EndOffset();
+
+                                if (++fieldState.length >= maxFieldLength)
+                                {
+                                    if (docState.infoStream != null)
+                                        docState.infoStream.WriteLine("maxFieldLength " + maxFieldLength + " reached for field " + fieldInfo.name + ", ignoring following tokens");
+                                    break;
+                                }
+                            }
+                            fieldState.offset = offsetEnd + 1;
+                        }
+                        finally
+                        {
+                            stream.Close();
+                        }
+                    }
+
+                    fieldState.boost *= field.GetBoost();
+                }
+            }
+
+            consumer.finish();
+            endConsumer.finish();
+        }
+    }
+}

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocInverterPerThread.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Token = Lucene.Net.Analysis.Token;
+
+namespace Lucene.Net.Index
+{
+    internal sealed class DocInverterPerThread : DocFieldConsumerPerThread
+    {
+        internal readonly DocInverter docInverter;
+        internal readonly InvertedDocConsumerPerThread consumer;
+        internal readonly InvertedDocEndConsumerPerThread endConsumer;
+        internal readonly Token localToken = new Token();
+        internal readonly DocumentsWriter.DocState docState;
+
+        internal readonly DocInverter.FieldInvertState fieldState = new DocInverter.FieldInvertState();
+
+        // Used to read a string value for a field
+        internal readonly ReusableStringReader stringReader = new ReusableStringReader();
+
+        public DocInverterPerThread(DocFieldProcessorPerThread docFieldProcessorPerThread, DocInverter docInverter)
+        {
+            this.docInverter = docInverter;
+            docState = docFieldProcessorPerThread.docState;
+            consumer = docInverter.consumer.addThread(this);
+            endConsumer = docInverter.endConsumer.addThread(this);
+        }
+
+        internal override void startDocument()
+        {
+            consumer.startDocument();
+            endConsumer.startDocument();
+        }
+
+        internal override DocumentsWriter.DocWriter finishDocument()
+        {
+            // TODO: allow endConsumer.finishDocument to also return
+            // a DocWriter
+            endConsumer.finishDocument();
+            return consumer.finishDocument();
+        }
+
+        internal override void abort()
+        {
+            try
+            {
+                consumer.abort();
+            }
+            finally
+            {
+                endConsumer.abort();
+            }
+        }
+
+        internal override DocFieldConsumerPerField addField(FieldInfo fi)
+        {
+            return new DocInverterPerField(this, fi);
+        }
+    }
+}