lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aro...@apache.org
Subject svn commit: r534192 [9/19] - in /incubator/lucene.net/trunk/C#: ./ src/ src/Demo/ src/Demo/DeleteFiles/ src/Demo/DemoLib/ src/Demo/DemoLib/HTML/ src/Demo/IndexFiles/ src/Demo/IndexHtml/ src/Demo/SearchFiles/ src/Lucene.Net/ src/Lucene.Net/Analysis/ src...
Date Tue, 01 May 2007 18:45:35 GMT
Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/MultiReader.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs Tue May  1 11:45:26 2007
@@ -17,7 +17,7 @@
 
 using System;
 using Document = Lucene.Net.Documents.Document;
-using Field = Lucene.Net.Documents.Field;
+using FieldSelector = Lucene.Net.Documents.FieldSelector;
 using Directory = Lucene.Net.Store.Directory;
 
 namespace Lucene.Net.Index
@@ -26,7 +26,7 @@
 	/// <summary>An IndexReader which reads multiple indexes, appending their content.
 	/// 
 	/// </summary>
-	/// <version>  $Id: MultiReader.java 355181 2005-12-08 19:53:06Z cutting $
+	/// <version>  $Id: MultiReader.java 499176 2007-01-23 22:54:40Z dnaber $
 	/// </version>
 	public class MultiReader : IndexReader
 	{
@@ -51,7 +51,7 @@
 		}
 		
 		/// <summary>Construct reading the named set of readers. </summary>
-		public /*internal*/ MultiReader(Directory directory, SegmentInfos sis, bool closeDirectory, IndexReader[] subReaders) : base(directory, sis, closeDirectory)
+		public MultiReader(Directory directory, SegmentInfos sis, bool closeDirectory, IndexReader[] subReaders) : base(directory, sis, closeDirectory)
 		{
 			Initialize(subReaders);
 		}
@@ -111,10 +111,10 @@
 			return maxDoc;
 		}
 		
-		public override Document Document(int n)
+		public override Document Document(int n, FieldSelector fieldSelector)
 		{
 			int i = ReaderIndex(n); // find segment num
-			return subReaders[i].Document(n - starts[i]); // dispatch to segment reader
+			return subReaders[i].Document(n - starts[i], fieldSelector); // dispatch to segment reader
 		}
 		
 		public override bool IsDeleted(int n)
@@ -182,7 +182,7 @@
 		}
 		
 		private byte[] ones;
-		private byte[] FakeNorms()
+		private byte[] fakeNorms()
 		{
 			if (ones == null)
 				ones = SegmentReader.CreateFakeNorms(MaxDoc());
@@ -197,7 +197,7 @@
 				if (bytes != null)
 					return bytes; // cache hit
 				if (!HasNorms(field))
-					return FakeNorms();
+					return fakeNorms();
 				
 				bytes = new byte[MaxDoc()];
 				for (int i = 0; i < subReaders.Length; i++)
@@ -213,13 +213,13 @@
 			{
 				byte[] bytes = (byte[]) normsCache[field];
 				if (bytes == null && !HasNorms(field))
-					bytes = FakeNorms();
+					bytes = fakeNorms();
 				if (bytes != null)
-				    // cache hit
+					// cache hit
 					Array.Copy(bytes, 0, result, offset, MaxDoc());
 				
 				for (int i = 0; i < subReaders.Length; i++)
-				    // read from segments
+					// read from segments
 					subReaders[i].Norms(field, result, offset + starts[i]);
 			}
 		}
@@ -259,12 +259,38 @@
 			return new MultiTermPositions(subReaders, starts);
 		}
 		
+		protected internal override void  SetDeleter(IndexFileDeleter deleter)
+		{
+			// Share deleter to our SegmentReaders:
+			this.deleter = deleter;
+			for (int i = 0; i < subReaders.Length; i++)
+				subReaders[i].SetDeleter(deleter);
+		}
+		
 		protected internal override void  DoCommit()
 		{
 			for (int i = 0; i < subReaders.Length; i++)
 				subReaders[i].Commit();
 		}
 		
+		internal override void  StartCommit()
+		{
+			base.StartCommit();
+			for (int i = 0; i < subReaders.Length; i++)
+			{
+				subReaders[i].StartCommit();
+			}
+		}
+		
+		internal override void  RollbackCommit()
+		{
+			base.RollbackCommit();
+			for (int i = 0; i < subReaders.Length; i++)
+			{
+				subReaders[i].RollbackCommit();
+			}
+		}
+		
 		protected internal override void  DoClose()
 		{
 			lock (this)
@@ -274,7 +300,7 @@
 			}
 		}
 		
-		/// <seealso cref="IndexReader.GetFieldNames(IndexReader.FieldOption)">
+		/// <seealso cref="IndexReader#GetFieldNames(IndexReader.FieldOption)">
 		/// </seealso>
 		public override System.Collections.ICollection GetFieldNames(IndexReader.FieldOption fieldNames)
 		{
@@ -420,18 +446,22 @@
 		
 		public virtual bool Next()
 		{
-			if (current != null && current.Next())
-			{
-				return true;
-			}
-			else if (pointer < readers.Length)
+			for (; ; )
 			{
-				base_Renamed = starts[pointer];
-				current = TermDocs(pointer++);
-				return Next();
+				if (current != null && current.Next())
+				{
+					return true;
+				}
+				else if (pointer < readers.Length)
+				{
+					base_Renamed = starts[pointer];
+					current = TermDocs(pointer++);
+				}
+				else
+				{
+					return false;
+				}
 			}
-			else
-				return false;
 		}
 		
 		/// <summary>Optimized implementation. </summary>
@@ -469,16 +499,23 @@
 			}
 		}
 		
-		/// <summary>As yet unoptimized implementation. </summary>
+		/* A Possible future optimization could skip entire segments */
 		public virtual bool SkipTo(int target)
 		{
-			do 
+			for (; ; )
 			{
-				if (!Next())
+				if (current != null && current.SkipTo(target - base_Renamed))
+				{
+					return true;
+				}
+				else if (pointer < readers.Length)
+				{
+					base_Renamed = starts[pointer];
+					current = TermDocs(pointer++);
+				}
+				else
 					return false;
 			}
-			while (target > Doc());
-			return true;
 		}
 		
 		private TermDocs TermDocs(int i)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/MultipleTermPositions.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs Tue May  1 11:45:26 2007
@@ -118,7 +118,7 @@
 		/// <summary> Creates a new <code>MultipleTermPositions</code> instance.
 		/// 
 		/// </summary>
-		/// <exception cref="IOException">
+		/// <exception cref=""> IOException
 		/// </exception>
 		public MultipleTermPositions(IndexReader indexReader, Term[] terms)
 		{

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Package.html
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/Package.html?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Package.html (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Package.html Tue May  1 11:45:26 2007
@@ -1,10 +1,10 @@
-<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
-<html>
-<head>
-   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-   <meta name="Author" content="Doug Cutting">
-</head>
-<body>
-Code to maintain and access indices.
-</body>
-</html>
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+   <meta name="Author" content="Doug Cutting">
+</head>
+<body>
+Code to maintain and access indices.
+</body>
+</html>

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ParallelReader.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs Tue May  1 11:45:26 2007
@@ -17,11 +17,14 @@
 
 using System;
 using Document = Lucene.Net.Documents.Document;
-using Field = Lucene.Net.Documents.Field;
+using Fieldable = Lucene.Net.Documents.Fieldable;
+using FieldSelector = Lucene.Net.Documents.FieldSelector;
+using FieldSelectorResult = Lucene.Net.Documents.FieldSelectorResult;
 
 namespace Lucene.Net.Index
 {
 	
+	
 	/// <summary>An IndexReader which reads multiple, parallel indexes.  Each index added
 	/// must have the same number of documents, but typically each contains
 	/// different fields.  Each document contains the union of the fields of all
@@ -43,6 +46,7 @@
 	{
 		private System.Collections.ArrayList readers = new System.Collections.ArrayList();
 		private System.Collections.SortedList fieldToReader = new System.Collections.SortedList();
+		private System.Collections.IDictionary readerToFields = new System.Collections.Hashtable();
 		private System.Collections.ArrayList storedFieldReaders = new System.Collections.ArrayList();
 		
 		private int maxDoc;
@@ -65,10 +69,10 @@
 		/// the IndexReaders.
 		/// 
 		/// </summary>
-		/// <throws>  IllegalArgumentException if not all indexes contain the same number  </throws>
+		/// <throws>  IllegalArgumentException if not all indexes contain the same number </throws>
 		/// <summary>     of documents
 		/// </summary>
-		/// <throws>  IllegalArgumentException if not all indexes have the same value  </throws>
+		/// <throws>  IllegalArgumentException if not all indexes have the same value </throws>
 		/// <summary>     of {@link IndexReader#MaxDoc()}
 		/// </summary>
 		public virtual void  Add(IndexReader reader, bool ignoreStoredFields)
@@ -87,10 +91,12 @@
 			if (reader.NumDocs() != numDocs)
 				throw new System.ArgumentException("All readers must have same numDocs: " + numDocs + "!=" + reader.NumDocs());
 			
-			System.Collections.IEnumerator i = reader.GetFieldNames(IndexReader.FieldOption.ALL).GetEnumerator();
+			System.Collections.ICollection fields = reader.GetFieldNames(IndexReader.FieldOption.ALL);
+			readerToFields[reader] = fields;
+			System.Collections.IEnumerator i = fields.GetEnumerator();
 			while (i.MoveNext())
 			{
-                System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) i.Current;
+				System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) i.Current;
 
 				// update fieldToReader map
 				System.String field = fi.Key.ToString();
@@ -147,15 +153,33 @@
 		}
 		
 		// append fields from storedFieldReaders
-		public override Document Document(int n)
+		public override Document Document(int n, FieldSelector fieldSelector)
 		{
 			Document result = new Document();
 			for (int i = 0; i < storedFieldReaders.Count; i++)
 			{
 				IndexReader reader = (IndexReader) storedFieldReaders[i];
-                foreach(Field field in reader.Document(n).Fields())
+				
+				bool include = (fieldSelector == null);
+				if (!include)
+				{
+					System.Collections.IEnumerator it = ((System.Collections.ICollection) readerToFields[reader]).GetEnumerator();
+					while (it.MoveNext())
+					{
+						if (fieldSelector.Accept((System.String) it.Current) != FieldSelectorResult.NO_LOAD)
+						{
+							include = true;
+							break;
+						}
+					}
+				}
+				if (include)
 				{
-					result.Add(field);
+					System.Collections.IEnumerator fieldIterator = reader.Document(n, fieldSelector).GetFields().GetEnumerator();
+					while (fieldIterator.MoveNext())
+					{
+						result.Add((Fieldable) fieldIterator.Current);
+					}
 				}
 			}
 			return result;
@@ -169,11 +193,9 @@
 			while (i.MoveNext())
 			{
 				System.Collections.DictionaryEntry e = (System.Collections.DictionaryEntry) i.Current;
-				//IndexReader reader = (IndexReader) e.Key;         // {{Aroush}} which is right, those two lines?
-				//System.String field = (System.String) e.Value;
-                System.String field = (System.String) e.Key;        // {{Aroush-2.0}} or those two lines?
-                IndexReader reader = (IndexReader) e.Value;
-                TermFreqVector vector = reader.GetTermFreqVector(n, field);
+				System.String field = (System.String) e.Key;
+				IndexReader reader = (IndexReader) e.Value;
+				TermFreqVector vector = reader.GetTermFreqVector(n, field);
 				if (vector != null)
 					results.Add(vector);
 			}
@@ -182,35 +204,35 @@
 		
 		public override TermFreqVector GetTermFreqVector(int n, System.String field)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[field]);
-            return reader == null ? null : reader.GetTermFreqVector(n, field);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[field]);
+			return reader == null ? null : reader.GetTermFreqVector(n, field);
+		}
 		
 		public override bool HasNorms(System.String field)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[field]);
-            return reader == null ? false : reader.HasNorms(field);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[field]);
+			return reader == null ? false : reader.HasNorms(field);
+		}
 		
 		public override byte[] Norms(System.String field)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[field]);
-            return reader == null ? null : reader.Norms(field);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[field]);
+			return reader == null ? null : reader.Norms(field);
+		}
 		
 		public override void  Norms(System.String field, byte[] result, int offset)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[field]);
-            if (reader != null)
-                reader.Norms(field, result, offset);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[field]);
+			if (reader != null)
+				reader.Norms(field, result, offset);
+		}
 		
 		protected internal override void  DoSetNorm(int n, System.String field, byte value_Renamed)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[field]);
-            if (reader != null)
-                reader.DoSetNorm(n, field, value_Renamed);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[field]);
+			if (reader != null)
+				reader.DoSetNorm(n, field, value_Renamed);
+		}
 		
 		public override TermEnum Terms()
 		{
@@ -224,9 +246,9 @@
 		
 		public override int DocFreq(Term term)
 		{
-            IndexReader reader = ((IndexReader) fieldToReader[term.Field()]);
-            return reader == null ? 0 : reader.DocFreq(term);
-        }
+			IndexReader reader = ((IndexReader) fieldToReader[term.Field()]);
+			return reader == null ? 0 : reader.DocFreq(term);
+		}
 		
 		public override TermDocs TermDocs(Term term)
 		{
@@ -264,24 +286,24 @@
 		}
 		
 		
-        public override System.Collections.ICollection GetFieldNames(IndexReader.FieldOption fieldNames)
+		public override System.Collections.ICollection GetFieldNames(IndexReader.FieldOption fieldNames)
 		{
-            System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
-            for (int i = 0; i < readers.Count; i++)
-            {
-                IndexReader reader = ((IndexReader) readers[i]);
-                System.Collections.ICollection names = reader.GetFieldNames(fieldNames);
-                for (System.Collections.IEnumerator iterator = names.GetEnumerator(); iterator.MoveNext(); )
-                {
-                    System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
-                    System.String s = fi.Key.ToString();
-                    if (fieldSet.ContainsKey(s) == false)
-                    {
-                        fieldSet.Add(s, s);
-                    }
-                }
-            }
-            return fieldSet;
+			System.Collections.Hashtable fieldSet = new System.Collections.Hashtable();
+			for (int i = 0; i < readers.Count; i++)
+			{
+				IndexReader reader = ((IndexReader) readers[i]);
+				System.Collections.ICollection names = reader.GetFieldNames(fieldNames);
+				for (System.Collections.IEnumerator iterator = names.GetEnumerator(); iterator.MoveNext(); )
+				{
+					System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
+					System.String s = fi.Key.ToString();
+					if (fieldSet.ContainsKey(s) == false)
+					{
+						fieldSet.Add(s, s);
+					}
+				}
+			}
+			return fieldSet;
 		}
 		
 		private class ParallelTermEnum : TermEnum
@@ -300,6 +322,7 @@
 				
 			}
 			private System.String field;
+			private System.Collections.IEnumerator fieldIterator;
 			private TermEnum termEnum;
 			
 			public ParallelTermEnum(ParallelReader enclosingInstance)
@@ -314,30 +337,37 @@
 			{
 				InitBlock(enclosingInstance);
 				field = term.Field();
-                IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[field]);
-                if (reader != null)
-                    termEnum = reader.Terms(term);
-            }
+				IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[field]);
+				if (reader != null)
+					termEnum = reader.Terms(term);
+			}
 			
 			public override bool Next()
 			{
 				if (termEnum == null)
 					return false;
 				
-				bool next = termEnum.Next();
-				
-				// still within field?
-				if (next && (System.Object) termEnum.Term().Field() == (System.Object) field)
+				// another term in this field?
+				if (termEnum.Next() && (System.Object) termEnum.Term().Field() == (System.Object) field)
 					return true; // yes, keep going
 				
 				termEnum.Close(); // close old termEnum
 				
-				// find the next field, if any
-				field = ((System.String) SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).GetKey(0));
-				if (field != null)
+				// find the next field with terms, if any
+				if (fieldIterator == null)
 				{
-					termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms();
-					return true;
+					fieldIterator = SupportClass.TailMap(Enclosing_Instance.fieldToReader, field).Keys.GetEnumerator();
+					System.Object generatedAux = fieldIterator.Current; // Skip field to get next one
+				}
+				while (fieldIterator.MoveNext())
+				{
+					field = ((System.String) fieldIterator.Current);
+					termEnum = ((IndexReader) Enclosing_Instance.fieldToReader[field]).Terms(new Term(field, ""));
+					Term term = termEnum.Term();
+					if (term != null && (System.Object) term.Field() == (System.Object) field)
+						return true;
+					else
+						termEnum.Close();
 				}
 				
 				return false; // no more fields
@@ -345,22 +375,24 @@
 			
 			public override Term Term()
 			{
-                if (termEnum == null)
-                    return null;
+				if (termEnum == null)
+					return null;
 				
-                return termEnum.Term();
+				return termEnum.Term();
 			}
+			
 			public override int DocFreq()
 			{
-                if (termEnum == null)
-                    return 0;
+				if (termEnum == null)
+					return 0;
 				
-                return termEnum.DocFreq();
+				return termEnum.DocFreq();
 			}
+			
 			public override void  Close()
 			{
-                if (termEnum != null)
-                    termEnum.Close();
+				if (termEnum != null)
+					termEnum.Close();
 			}
 		}
 		
@@ -403,9 +435,9 @@
 			
 			public virtual void  Seek(Term term)
 			{
-                IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[term.Field()]);
-                termDocs = reader != null ? reader.TermDocs(term) : null;
-            }
+				IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[term.Field()]);
+				termDocs = reader != null ? reader.TermDocs(term) : null;
+			}
 			
 			public virtual void  Seek(TermEnum termEnum)
 			{
@@ -414,32 +446,32 @@
 			
 			public virtual bool Next()
 			{
-                if (termDocs == null)
-                    return false;
+				if (termDocs == null)
+					return false;
 				
-                return termDocs.Next();
+				return termDocs.Next();
 			}
 			
 			public virtual int Read(int[] docs, int[] freqs)
 			{
-                if (termDocs == null)
-                    return 0;
+				if (termDocs == null)
+					return 0;
 				
-                return termDocs.Read(docs, freqs);
+				return termDocs.Read(docs, freqs);
 			}
 			
 			public virtual bool SkipTo(int target)
 			{
-                if (termDocs == null)
-                    return false;
+				if (termDocs == null)
+					return false;
 				
-                return termDocs.SkipTo(target);
+				return termDocs.SkipTo(target);
 			}
 			
 			public virtual void  Close()
 			{
-                if (termDocs != null)
-                    termDocs.Close();
+				if (termDocs != null)
+					termDocs.Close();
 			}
 		}
 		
@@ -471,9 +503,9 @@
 			
 			public override void  Seek(Term term)
 			{
-                IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[term.Field()]);
-                termDocs = reader != null ? reader.TermPositions(term) : null;
-            }
+				IndexReader reader = ((IndexReader) Enclosing_Instance.fieldToReader[term.Field()]);
+				termDocs = reader != null ? reader.TermPositions(term) : null;
+			}
 			
 			public virtual int NextPosition()
 			{

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentInfo.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs Tue May  1 11:45:26 2007
@@ -17,21 +17,424 @@
 
 using System;
 using Directory = Lucene.Net.Store.Directory;
+using IndexOutput = Lucene.Net.Store.IndexOutput;
+using IndexInput = Lucene.Net.Store.IndexInput;
 
 namespace Lucene.Net.Index
 {
 	
-	public sealed class SegmentInfo
+	sealed public class SegmentInfo : System.ICloneable
 	{
 		public System.String name; // unique name in dir
 		public int docCount; // number of docs in seg
 		public Directory dir; // where segment resides
 		
+		private bool preLockless; // true if this is a segments file written before
+		// lock-less commits (2.1)
+		
+		private long delGen; // current generation of del file; -1 if there
+		// are no deletes; 0 if it's a pre-2.1 segment
+		// (and we must check filesystem); 1 or higher if
+		// there are deletes at generation N
+		
+		private long[] normGen; // current generations of each field's norm file.
+		// If this array is null, we must check filesystem
+		// when preLockLess is true.  Else,
+		// there are no separate norms
+		
+		private sbyte isCompoundFile; // -1 if it is not; 1 if it is; 0 if it's
+		// pre-2.1 (ie, must check file system to see
+		// if <name>.cfs and <name>.nrm exist)         
+		
+		private bool hasSingleNormFile; // true if this segment maintains norms in a single file; 
+		// false otherwise
+		// this is currently false for segments populated by DocumentWriter
+		// and true for newly created merged segments (both
+		// compound and non compound).
+		
 		public SegmentInfo(System.String name, int docCount, Directory dir)
 		{
 			this.name = name;
 			this.docCount = docCount;
 			this.dir = dir;
+			delGen = - 1;
+			isCompoundFile = 0;
+			preLockless = true;
+			hasSingleNormFile = false;
+		}
+		
+		public SegmentInfo(System.String name, int docCount, Directory dir, bool isCompoundFile, bool hasSingleNormFile) : this(name, docCount, dir)
+		{
+			this.isCompoundFile = (sbyte) (isCompoundFile ? 1 : - 1);
+			this.hasSingleNormFile = hasSingleNormFile;
+			preLockless = false;
+		}
+		
+		/// <summary> Copy everything from src SegmentInfo into our instance.</summary>
+		internal void  Reset(SegmentInfo src)
+		{
+			name = src.name;
+			docCount = src.docCount;
+			dir = src.dir;
+			preLockless = src.preLockless;
+			delGen = src.delGen;
+			if (src.normGen == null)
+			{
+				normGen = null;
+			}
+			else
+			{
+				normGen = new long[src.normGen.Length];
+				Array.Copy(src.normGen, 0, normGen, 0, src.normGen.Length);
+			}
+			isCompoundFile = src.isCompoundFile;
+			hasSingleNormFile = src.hasSingleNormFile;
+		}
+		
+		/// <summary> Construct a new SegmentInfo instance by reading a
+		/// previously saved SegmentInfo from input.
+		/// 
+		/// </summary>
+		/// <param name="dir">directory to load from
+		/// </param>
+		/// <param name="format">format of the segments info file
+		/// </param>
+		/// <param name="input">input handle to read segment info from
+		/// </param>
+		public SegmentInfo(Directory dir, int format, IndexInput input)
+		{
+			this.dir = dir;
+			name = input.ReadString();
+			docCount = input.ReadInt();
+			if (format <= SegmentInfos.FORMAT_LOCKLESS)
+			{
+				delGen = input.ReadLong();
+				if (format <= SegmentInfos.FORMAT_SINGLE_NORM_FILE)
+				{
+					hasSingleNormFile = (1 == input.ReadByte());
+				}
+				else
+				{
+					hasSingleNormFile = false;
+				}
+				int numNormGen = input.ReadInt();
+				if (numNormGen == - 1)
+				{
+					normGen = null;
+				}
+				else
+				{
+					normGen = new long[numNormGen];
+					for (int j = 0; j < numNormGen; j++)
+					{
+						normGen[j] = input.ReadLong();
+					}
+				}
+				isCompoundFile = (sbyte) input.ReadByte();
+				preLockless = isCompoundFile == 0;
+			}
+			else
+			{
+				delGen = 0;
+				normGen = null;
+				isCompoundFile = 0;
+				preLockless = true;
+				hasSingleNormFile = false;
+			}
+		}
+		
+		internal void  SetNumFields(int numFields)
+		{
+			if (normGen == null)
+			{
+				// normGen is null if we loaded a pre-2.1 segment
+				// file, or, if this segments file hasn't had any
+				// norms set against it yet:
+				normGen = new long[numFields];
+				
+				if (!preLockless)
+				{
+					// This is a FORMAT_LOCKLESS segment, which means
+					// there are no norms:
+					for (int i = 0; i < numFields; i++)
+					{
+						normGen[i] = - 1;
+					}
+				}
+			}
+		}
+		
+		internal bool HasDeletions()
+		{
+			// Cases:
+			//
+			//   delGen == -1: this means this segment was written
+			//     by the LOCKLESS code and for certain does not have
+			//     deletions yet
+			//
+			//   delGen == 0: this means this segment was written by
+			//     pre-LOCKLESS code which means we must check
+			//     directory to see if .del file exists
+			//
+			//   delGen > 0: this means this segment was written by
+			//     the LOCKLESS code and for certain has
+			//     deletions
+			//
+			if (delGen == - 1)
+			{
+				return false;
+			}
+			else if (delGen > 0)
+			{
+				return true;
+			}
+			else
+			{
+				return dir.FileExists(GetDelFileName());
+			}
+		}
+		
+		internal void  AdvanceDelGen()
+		{
+			// delGen 0 is reserved for pre-LOCKLESS format
+			if (delGen == - 1)
+			{
+				delGen = 1;
+			}
+			else
+			{
+				delGen++;
+			}
+		}
+		
+		internal void  ClearDelGen()
+		{
+			delGen = - 1;
+		}
+		
+		public System.Object Clone()
+		{
+			SegmentInfo si = new SegmentInfo(name, docCount, dir);
+			si.isCompoundFile = isCompoundFile;
+			si.delGen = delGen;
+			si.preLockless = preLockless;
+			si.hasSingleNormFile = hasSingleNormFile;
+			if (normGen != null)
+			{
+				si.normGen = new long[normGen.Length];
+				normGen.CopyTo(si.normGen, 0);
+			}
+			return si;
+		}
+		
+		internal System.String GetDelFileName()
+		{
+			if (delGen == - 1)
+			{
+				// In this case we know there is no deletion filename
+				// against this segment
+				return null;
+			}
+			else
+			{
+				// If delGen is 0, it's the pre-lockless-commit file format
+				return IndexFileNames.FileNameFromGeneration(name, ".del", delGen);
+			}
+		}
+		
+		/// <summary> Returns true if this field for this segment has saved a separate norms file (_<segment>_N.sX).
+		/// 
+		/// </summary>
+		/// <param name="fieldNumber">the field index to check
+		/// </param>
+		internal bool HasSeparateNorms(int fieldNumber)
+		{
+			if ((normGen == null && preLockless) || (normGen != null && normGen[fieldNumber] == 0))
+			{
+				// Must fallback to directory file exists check:
+				System.String fileName = name + ".s" + fieldNumber;
+				return dir.FileExists(fileName);
+			}
+			else if (normGen == null || normGen[fieldNumber] == - 1)
+			{
+				return false;
+			}
+			else
+			{
+				return true;
+			}
+		}
+		
+		/// <summary> Returns true if any fields in this segment have separate norms.</summary>
+		internal bool HasSeparateNorms()
+		{
+			if (normGen == null)
+			{
+				if (!preLockless)
+				{
+					// This means we were created w/ LOCKLESS code and no
+					// norms are written yet:
+					return false;
+				}
+				else
+				{
+					// This means this segment was saved with pre-LOCKLESS
+					// code.  So we must fallback to the original
+					// directory list check:
+					System.String[] result = dir.List();
+					System.String pattern;
+					pattern = name + ".s";
+					int patternLength = pattern.Length;
+					for (int i = 0; i < result.Length; i++)
+					{
+						if (result[i].StartsWith(pattern) && System.Char.IsDigit(result[i][patternLength]))
+							return true;
+					}
+					return false;
+				}
+			}
+			else
+			{
+				// This means this segment was saved with LOCKLESS
+				// code so we first check whether any normGen's are >
+				// 0 (meaning they definitely have separate norms):
+				for (int i = 0; i < normGen.Length; i++)
+				{
+					if (normGen[i] > 0)
+					{
+						return true;
+					}
+				}
+				// Next we look for any == 0.  These cases were
+				// pre-LOCKLESS and must be checked in directory:
+				for (int i = 0; i < normGen.Length; i++)
+				{
+					if (normGen[i] == 0)
+					{
+						if (HasSeparateNorms(i))
+						{
+							return true;
+						}
+					}
+				}
+			}
+			
+			return false;
+		}
+		
+		/// <summary> Increment the generation count for the norms file for
+		/// this field.
+		/// 
+		/// </summary>
+		/// <param name="fieldIndex">field whose norm file will be rewritten
+		/// </param>
+		internal void  AdvanceNormGen(int fieldIndex)
+		{
+			if (normGen[fieldIndex] == - 1)
+			{
+				normGen[fieldIndex] = 1;
+			}
+			else
+			{
+				normGen[fieldIndex]++;
+			}
+		}
+		
+		/// <summary> Get the file name for the norms file for this field.
+		/// 
+		/// </summary>
+		/// <param name="number">field index
+		/// </param>
+		internal System.String GetNormFileName(int number)
+		{
+			System.String prefix;
+			
+			long gen;
+			if (normGen == null)
+			{
+				gen = 0;
+			}
+			else
+			{
+				gen = normGen[number];
+			}
+			
+			if (HasSeparateNorms(number))
+			{
+				// case 1: separate norm
+				prefix = ".s";
+				return IndexFileNames.FileNameFromGeneration(name, prefix + number, gen);
+			}
+			
+			if (hasSingleNormFile)
+			{
+				// case 2: lockless (or nrm file exists) - single file for all norms 
+				prefix = "." + IndexFileNames.NORMS_EXTENSION;
+				return IndexFileNames.FileNameFromGeneration(name, prefix, 0);
+			}
+			
+			// case 3: norm file for each field
+			prefix = ".f";
+			return IndexFileNames.FileNameFromGeneration(name, prefix + number, 0);
+		}
+		
+		/// <summary> Mark whether this segment is stored as a compound file.
+		/// 
+		/// </summary>
+		/// <param name="isCompoundFile">true if this is a compound file;
+		/// else, false
+		/// </param>
+		internal void  SetUseCompoundFile(bool isCompoundFile)
+		{
+			if (isCompoundFile)
+			{
+				this.isCompoundFile = 1;
+			}
+			else
+			{
+				this.isCompoundFile = - 1;
+			}
+		}
+		
+		/// <summary> Returns true if this segment is stored as a compound
+		/// file; else, false.
+		/// </summary>
+		internal bool GetUseCompoundFile()
+		{
+			if (isCompoundFile == - 1)
+			{
+				return false;
+			}
+			else if (isCompoundFile == 1)
+			{
+				return true;
+			}
+			else
+			{
+				return dir.FileExists(name + ".cfs");
+			}
+		}
+		
+		/// <summary> Save this segment's info.</summary>
+		internal void  Write(IndexOutput output)
+		{
+			output.WriteString(name);
+			output.WriteInt(docCount);
+			output.WriteLong(delGen);
+			output.WriteByte((byte) (hasSingleNormFile ? 1 : 0));
+			if (normGen == null)
+			{
+				output.WriteInt(- 1);
+			}
+			else
+			{
+				output.WriteInt(normGen.Length);
+				for (int j = 0; j < normGen.Length; j++)
+				{
+					output.WriteLong(normGen[j]);
+				}
+			}
+			output.WriteByte((byte) isCompoundFile);
 		}
 	}
 }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentInfos.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs Tue May  1 11:45:26 2007
@@ -19,7 +19,6 @@
 using Directory = Lucene.Net.Store.Directory;
 using IndexInput = Lucene.Net.Store.IndexInput;
 using IndexOutput = Lucene.Net.Store.IndexOutput;
-using Constants = Lucene.Net.Util.Constants;
 
 namespace Lucene.Net.Index
 {
@@ -27,26 +26,238 @@
 	[Serializable]
 	public sealed class SegmentInfos : System.Collections.ArrayList
 	{
+		private class AnonymousClassFindSegmentsFile:FindSegmentsFile
+		{
+			private void  InitBlock(SegmentInfos enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private SegmentInfos enclosingInstance;
+			public SegmentInfos Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			internal AnonymousClassFindSegmentsFile(SegmentInfos enclosingInstance, Lucene.Net.Store.Directory Param1):base(Param1)
+			{
+				InitBlock(enclosingInstance);
+			}
+			
+			public override System.Object DoBody(System.String segmentFileName)
+			{
+				Enclosing_Instance.Read(directory, segmentFileName);
+				return null;
+			}
+		}
+		private class AnonymousClassFindSegmentsFile1:FindSegmentsFile
+		{
+			internal AnonymousClassFindSegmentsFile1(Lucene.Net.Store.Directory Param1):base(Param1)
+			{
+			}
+			public override System.Object DoBody(System.String segmentFileName)
+			{
+				
+				IndexInput input = directory.OpenInput(segmentFileName);
+				
+				int format = 0;
+				long version = 0;
+				try
+				{
+					format = input.ReadInt();
+					if (format < 0)
+					{
+						if (format < Lucene.Net.Index.SegmentInfos.FORMAT_SINGLE_NORM_FILE)
+							throw new System.IO.IOException("Unknown format version: " + format);
+						version = input.ReadLong(); // read version
+					}
+				}
+				finally
+				{
+					input.Close();
+				}
+				
+				if (format < 0)
+					return (long) version;
+				
+				// We cannot be sure about the format of the file.
+				// Therefore we have to read the whole file and cannot simply seek to the version entry.
+				SegmentInfos sis = new SegmentInfos();
+				sis.Read(directory, segmentFileName);
+				return (long) sis.GetVersion();
+			}
+		}
 		
 		/// <summary>The file format version, a negative number. </summary>
 		/* Works since counter, the old 1st entry, is always >= 0 */
 		public const int FORMAT = - 1;
 		
+		/// <summary>This format adds details used for lockless commits.  It differs
+		/// slightly from the previous format in that file names
+		/// are never re-used (write once).  Instead, each file is
+		/// written to the next generation.  For example,
+		/// segments_1, segments_2, etc.  This allows us to not use
+		/// a commit lock.  See <a
+		/// href="http://lucene.apache.org/java/docs/fileformats.html">file
+		/// formats</a> for details.
+		/// </summary>
+		public const int FORMAT_LOCKLESS = - 2;
+		
+		/// <summary>This is the current file format written.  It adds a
+		/// "hasSingleNormFile" flag into each segment info.
+		/// See <a href="http://issues.apache.org/jira/browse/LUCENE-756">LUCENE-756</a>
+		/// for details.
+		/// </summary>
+		public const int FORMAT_SINGLE_NORM_FILE = - 3;
+		
 		public int counter = 0; // used to name new segments
 		/// <summary> counts how often the index has been changed by adding or deleting docs.
 		/// starting with the current time in milliseconds forces to create unique version numbers.
 		/// </summary>
-		private long version = System.DateTime.Now.Ticks;
+		private long version = System.DateTime.Now.Millisecond;
+		
+		private long generation = 0; // generation of the "segments_N" for the next commit
+		private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
+		// or wrote; this is normally the same as generation except if
+		// there was an IOException that had interrupted a commit
+		
+		/// <summary> If non-null, information about loading segments_N files</summary>
+		/// <seealso cref="#setInfoStream.">
+		/// </seealso>
+		private static System.IO.TextWriter infoStream;
 		
 		public SegmentInfo Info(int i)
 		{
 			return (SegmentInfo) this[i];
 		}
 		
-		public void  Read(Directory directory)
+		/// <summary> Get the generation (N) of the current segments_N file
+		/// from a list of files.
+		/// 
+		/// </summary>
+		/// <param name="files">-- array of file names to check
+		/// </param>
+		public static long GetCurrentSegmentGeneration(System.String[] files)
 		{
+			if (files == null)
+			{
+				return - 1;
+			}
+			long max = - 1;
+			int prefixLen = IndexFileNames.SEGMENTS.Length + 1;
+			for (int i = 0; i < files.Length; i++)
+			{
+				System.String file = files[i];
+				if (file.StartsWith(IndexFileNames.SEGMENTS) && !file.Equals(IndexFileNames.SEGMENTS_GEN))
+				{
+					if (file.Equals(IndexFileNames.SEGMENTS))
+					{
+						// Pre lock-less commits:
+						if (max == - 1)
+						{
+							max = 0;
+						}
+					}
+					else
+					{
+						long v = System.Convert.ToInt64(file.Substring(prefixLen), 16);
+						if (v > max)
+						{
+							max = v;
+						}
+					}
+				}
+			}
+			return max;
+		}
+		
+		/// <summary> Get the generation (N) of the current segments_N file
+		/// in the directory.
+		/// 
+		/// </summary>
+		/// <param name="directory">-- directory to search for the latest segments_N file
+		/// </param>
+		public static long GetCurrentSegmentGeneration(Directory directory)
+		{
+			System.String[] files = directory.List();
+			if (files == null)
+			{
+				throw new System.IO.IOException("Cannot read directory " + directory);
+			}
+			return GetCurrentSegmentGeneration(files);
+		}
+		
+		/// <summary> Get the filename of the current segments_N file
+		/// from a list of files.
+		/// 
+		/// </summary>
+		/// <param name="files">-- array of file names to check
+		/// </param>
+		
+		public static System.String GetCurrentSegmentFileName(System.String[] files)
+		{
+			return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(files));
+		}
+		
+		/// <summary> Get the filename of the current segments_N file
+		/// in the directory.
+		/// 
+		/// </summary>
+		/// <param name="directory">-- directory to search for the latest segments_N file
+		/// </param>
+		public static System.String GetCurrentSegmentFileName(Directory directory)
+		{
+			return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(directory));
+		}
+		
+		/// <summary> Get the segments_N filename in use by this segment infos.</summary>
+		public System.String GetCurrentSegmentFileName()
+		{
+			return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", lastGeneration);
+		}
+		
+		/// <summary> Get the next segments_N filename that will be written.</summary>
+		public System.String GetNextSegmentFileName()
+		{
+			long nextGeneration;
+			
+			if (generation == - 1)
+			{
+				nextGeneration = 1;
+			}
+			else
+			{
+				nextGeneration = generation + 1;
+			}
+			return IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", nextGeneration);
+		}
+		
+		/// <summary> Read a particular segmentFileName.  Note that this may
+		/// throw an IOException if a commit is in process.
+		/// 
+		/// </summary>
+		/// <param name="directory">-- directory containing the segments file
+		/// </param>
+		/// <param name="segmentFileName">-- segment file to load
+		/// </param>
+		public void  Read(Directory directory, System.String segmentFileName)
+		{
+			bool success = false;
+			
+			IndexInput input = directory.OpenInput(segmentFileName);
+			
+			if (segmentFileName.Equals(IndexFileNames.SEGMENTS))
+			{
+				generation = 0;
+			}
+			else
+			{
+				generation = System.Convert.ToInt64(segmentFileName.Substring(1 + IndexFileNames.SEGMENTS.Length), 16);
+			}
+			lastGeneration = generation;
 			
-			IndexInput input = directory.OpenInput(IndexFileNames.SEGMENTS);
 			try
 			{
 				int format = input.ReadInt();
@@ -54,7 +265,7 @@
 				{
 					// file contains explicit format info
 					// check that it is a format we can understand
-					if (format < FORMAT)
+					if (format < FORMAT_SINGLE_NORM_FILE)
 						throw new System.IO.IOException("Unknown format version: " + format);
 					version = input.ReadLong(); // read version
 					counter = input.ReadInt(); // read counter
@@ -68,40 +279,69 @@
 				for (int i = input.ReadInt(); i > 0; i--)
 				{
 					// read segmentInfos
-					SegmentInfo si = new SegmentInfo(input.ReadString(), input.ReadInt(), directory);
-					Add(si);
+					Add(new SegmentInfo(directory, format, input));
 				}
 				
 				if (format >= 0)
 				{
 					// in old format the version number may be at the end of the file
 					if (input.GetFilePointer() >= input.Length())
-						version = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+						version = System.DateTime.Now.Millisecond;
 					// old file format without version number
 					else
 						version = input.ReadLong(); // read version
 				}
+				success = true;
 			}
 			finally
 			{
 				input.Close();
+				if (!success)
+				{
+					// Clear any segment infos we had loaded so we
+					// have a clean slate on retry:
+					Clear();
+				}
 			}
 		}
+		/// <summary> This version of read uses the retry logic (for lock-less
+		/// commits) to find the right segments file to load.
+		/// </summary>
+		public void  Read(Directory directory)
+		{
+			
+			generation = lastGeneration = - 1;
+			
+			new AnonymousClassFindSegmentsFile(this, directory).run();
+		}
 		
 		public void  Write(Directory directory)
 		{
-			IndexOutput output = directory.CreateOutput("segments.new");
+			
+			System.String segmentFileName = GetNextSegmentFileName();
+			
+			// Always advance the generation on write:
+			if (generation == - 1)
+			{
+				generation = 1;
+			}
+			else
+			{
+				generation++;
+			}
+			
+			IndexOutput output = directory.CreateOutput(segmentFileName);
+			
 			try
 			{
-				output.WriteInt(FORMAT); // write FORMAT
-				output.WriteLong(++version); // every write changes the index
+				output.WriteInt(FORMAT_SINGLE_NORM_FILE); // write FORMAT
+				output.WriteLong(++version); // every write changes
+				// the index
 				output.WriteInt(counter); // write counter
 				output.WriteInt(Count); // write infos
 				for (int i = 0; i < Count; i++)
 				{
-					SegmentInfo si = Info(i);
-					output.WriteString(si.name);
-					output.WriteInt(si.docCount);
+					Info(i).Write(output);
 				}
 			}
 			finally
@@ -109,8 +349,41 @@
 				output.Close();
 			}
 			
-			// install new segment info
-			directory.RenameFile("segments.new", IndexFileNames.SEGMENTS);
+			try
+			{
+				output = directory.CreateOutput(IndexFileNames.SEGMENTS_GEN);
+				try
+				{
+					output.WriteInt(FORMAT_LOCKLESS);
+					output.WriteLong(generation);
+					output.WriteLong(generation);
+				}
+				finally
+				{
+					output.Close();
+				}
+			}
+			catch (System.IO.IOException e)
+			{
+				// It's OK if we fail to write this file since it's
+				// used only as one of the retry fallbacks.
+			}
+			
+			lastGeneration = generation;
+		}
+		
+		/// <summary> Returns a copy of this instance, also copying each
+		/// SegmentInfo.
+		/// </summary>
+		
+		public override System.Object Clone()
+		{
+			SegmentInfos sis = (SegmentInfos) base.Clone();
+			for (int i = 0; i < sis.Count; i++)
+			{
+				sis[i] = ((SegmentInfo) sis[i]).Clone();
+			}
+			return sis;
 		}
 		
 		/// <summary> version number when this SegmentInfos was generated.</summary>
@@ -123,33 +396,343 @@
 		public static long ReadCurrentVersion(Directory directory)
 		{
 			
-			IndexInput input = directory.OpenInput(IndexFileNames.SEGMENTS);
-			int format = 0;
-			long version = 0;
-			try
+			return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).run());
+		}
+		
+		/// <summary>If non-null, information about retries when loading
+		/// the segments file will be printed to this.
+		/// </summary>
+		public static void  SetInfoStream(System.IO.TextWriter infoStream)
+		{
+			SegmentInfos.infoStream = infoStream;
+		}
+		
+		/* Advanced configuration of retry logic in loading
+		segments_N file */
+		private static int defaultGenFileRetryCount = 10;
+		private static int defaultGenFileRetryPauseMsec = 50;
+		private static int defaultGenLookaheadCount = 10;
+		
+		/// <summary> Advanced: set how many times to try loading the
+		/// segments.gen file contents to determine current segment
+		/// generation.  This file is only referenced when the
+		/// primary method (listing the directory) fails.
+		/// </summary>
+		public static void  SetDefaultGenFileRetryCount(int count)
+		{
+			defaultGenFileRetryCount = count;
+		}
+		
+		/// <seealso cref="#setDefaultGenFileRetryCount">
+		/// </seealso>
+		public static int GetDefaultGenFileRetryCount()
+		{
+			return defaultGenFileRetryCount;
+		}
+		
+		/// <summary> Advanced: set how many milliseconds to pause in between
+		/// attempts to load the segments.gen file.
+		/// </summary>
+		public static void  SetDefaultGenFileRetryPauseMsec(int msec)
+		{
+			defaultGenFileRetryPauseMsec = msec;
+		}
+		
+		/// <seealso cref="#setDefaultGenFileRetryPauseMsec">
+		/// </seealso>
+		public static int GetDefaultGenFileRetryPauseMsec()
+		{
+			return defaultGenFileRetryPauseMsec;
+		}
+		
+		/// <summary> Advanced: set how many times to try incrementing the
+		/// gen when loading the segments file.  This only runs if
+		/// the primary (listing directory) and secondary (opening
+		/// segments.gen file) methods fail to find the segments
+		/// file.
+		/// </summary>
+		public static void  SetDefaultGenLookaheadCount(int count)
+		{
+			defaultGenLookaheadCount = count;
+		}
+		/// <seealso cref="#setDefaultGenLookaheadCount">
+		/// </seealso>
+		public static int GetDefaultGenLookahedCount()
+		{
+			return defaultGenLookaheadCount;
+		}
+		
+		/// <seealso cref="#setInfoStream">
+		/// </seealso>
+		public static System.IO.TextWriter GetInfoStream()
+		{
+			return infoStream;
+		}
+		
+		private static void  Message(System.String message)
+		{
+			if (infoStream != null)
 			{
-				format = input.ReadInt();
-				if (format < 0)
-				{
-					if (format < FORMAT)
-						throw new System.IO.IOException("Unknown format version: " + format);
-					version = input.ReadLong(); // read version
-				}
+				infoStream.WriteLine(SupportClass.ThreadClass.Current().Name + ": " + message);
 			}
-			finally
+		}
+		
+		/// <summary> Utility class for executing code that needs to do
+		/// something with the current segments file.  This is
+		/// necessary with lock-less commits because from the time
+		/// you locate the current segments file name, until you
+		/// actually open it, read its contents, or check modified
+		/// time, etc., it could have been deleted due to a writer
+		/// commit finishing.
+		/// </summary>
+		public abstract class FindSegmentsFile
+		{
+			
+			internal System.IO.FileInfo fileDirectory;
+			internal Directory directory;
+			
+			public FindSegmentsFile(System.IO.FileInfo directory)
 			{
-				input.Close();
+				this.fileDirectory = directory;
 			}
 			
-			if (format < 0)
-				return version;
+			public FindSegmentsFile(Directory directory)
+			{
+				this.directory = directory;
+			}
 			
-			// We cannot be sure about the format of the file.
-			// Therefore we have to read the whole file and cannot simply seek to the version entry.
+			public System.Object run()
+			{
+				System.String segmentFileName = null;
+				long lastGen = - 1;
+				long gen = 0;
+				int genLookaheadCount = 0;
+				System.IO.IOException exc = null;
+				bool retry = false;
+				
+				int method = 0;
+				
+				// Loop until we succeed in calling doBody() without
+				// hitting an IOException.  An IOException most likely
+				// means a commit was in process and has finished, in
+				// the time it took us to load the now-old infos files
+				// (and segments files).  It's also possible it's a
+				// true error (corrupt index).  To distinguish these,
+				// on each retry we must see "forward progress" on
+				// which generation we are trying to load.  If we
+				// don't, then the original error is real and we throw
+				// it.
+				
+				// We have three methods for determining the current
+				// generation.  We try each in sequence.
+				
+				while (true)
+				{
+					
+					// Method 1: list the directory and use the highest
+					// segments_N file.  This method works well as long
+					// as there is no stale caching on the directory
+					// contents:
+					System.String[] files = null;
+					
+					if (0 == method)
+					{
+						if (directory != null)
+						{
+							files = directory.List();
+						}
+						else
+						{
+							files = System.IO.Directory.GetFileSystemEntries(fileDirectory.FullName);
+                            for (int i = 0; i < files.Length; i++)
+                            {
+                                System.IO.FileInfo fi = new System.IO.FileInfo(files[i]);
+                                files[i] = fi.Name;
+                            }
+                        }
+						
+						gen = Lucene.Net.Index.SegmentInfos.GetCurrentSegmentGeneration(files);
+						
+						if (gen == - 1)
+						{
+							System.String s = "";
+							for (int i = 0; i < files.Length; i++)
+							{
+								s += (" " + files[i]);
+							}
+							throw new System.IO.FileNotFoundException("no segments* file found: files:" + s);
+						}
+					}
+					
+					// Method 2 (fallback if Method 1 isn't reliable):
+					// if the directory listing seems to be stale, then
+					// try loading the "segments.gen" file.
+					if (1 == method || (0 == method && lastGen == gen && retry))
+					{
+						
+						method = 1;
+						
+						for (int i = 0; i < Lucene.Net.Index.SegmentInfos.defaultGenFileRetryCount; i++)
+						{
+							IndexInput genInput = null;
+							try
+							{
+								genInput = directory.OpenInput(IndexFileNames.SEGMENTS_GEN);
+							}
+							catch (System.IO.IOException e)
+							{
+								Lucene.Net.Index.SegmentInfos.Message("segments.gen open: IOException " + e);
+							}
+							if (genInput != null)
+							{
+								
+								try
+								{
+									int version = genInput.ReadInt();
+									if (version == Lucene.Net.Index.SegmentInfos.FORMAT_LOCKLESS)
+									{
+										long gen0 = genInput.ReadLong();
+										long gen1 = genInput.ReadLong();
+										Lucene.Net.Index.SegmentInfos.Message("fallback check: " + gen0 + "; " + gen1);
+										if (gen0 == gen1)
+										{
+											// The file is consistent.
+											if (gen0 > gen)
+											{
+												Lucene.Net.Index.SegmentInfos.Message("fallback to '" + IndexFileNames.SEGMENTS_GEN + "' check: now try generation " + gen0 + " > " + gen);
+												gen = gen0;
+											}
+											break;
+										}
+									}
+								}
+								catch (System.IO.IOException err2)
+								{
+									// will retry
+								}
+								finally
+								{
+									genInput.Close();
+								}
+							}
+							try
+							{
+								System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * Lucene.Net.Index.SegmentInfos.defaultGenFileRetryPauseMsec));
+							}
+							catch (System.Threading.ThreadInterruptedException e)
+							{
+								// will retry
+							}
+						}
+					}
+					
+					// Method 3 (fallback if Methods 2 & 3 are not
+					// reliable): since both directory cache and file
+					// contents cache seem to be stale, just advance the
+					// generation.
+					if (2 == method || (1 == method && lastGen == gen && retry))
+					{
+						
+						method = 2;
+						
+						if (genLookaheadCount < Lucene.Net.Index.SegmentInfos.defaultGenLookaheadCount)
+						{
+							gen++;
+							genLookaheadCount++;
+							Lucene.Net.Index.SegmentInfos.Message("look ahead increment gen to " + gen);
+						}
+					}
+					
+					if (lastGen == gen)
+					{
+						
+						// This means we're about to try the same
+						// segments_N last tried.  This is allowed,
+						// exactly once, because writer could have been in
+						// the process of writing segments_N last time.
+						
+						if (retry)
+						{
+							// OK, we've tried the same segments_N file
+							// twice in a row, so this must be a real
+							// error.  We throw the original exception we
+							// got.
+							throw exc;
+						}
+						else
+						{
+							retry = true;
+						}
+					}
+					else
+					{
+						// Segment file has advanced since our last loop, so
+						// reset retry:
+						retry = false;
+					}
+					
+					lastGen = gen;
+					
+					segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
+					
+					try
+					{
+						System.Object v = DoBody(segmentFileName);
+						if (exc != null)
+						{
+							Lucene.Net.Index.SegmentInfos.Message("success on " + segmentFileName);
+						}
+						return v;
+					}
+					catch (System.IO.IOException err)
+					{
+						
+						// Save the original root cause:
+						if (exc == null)
+						{
+							exc = err;
+						}
+						
+						Lucene.Net.Index.SegmentInfos.Message("primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retry=" + retry + "; gen = " + gen);
+						
+						if (!retry && gen > 1)
+						{
+							
+							// This is our first time trying this segments
+							// file (because retry is false), and, there is
+							// possibly a segments_(N-1) (because gen > 1).
+							// So, check if the segments_(N-1) exists and
+							// try it if so:
+							System.String prevSegmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
+							
+							if (directory.FileExists(prevSegmentFileName))
+							{
+								Lucene.Net.Index.SegmentInfos.Message("fallback to prior segment file '" + prevSegmentFileName + "'");
+								try
+								{
+									System.Object v = DoBody(prevSegmentFileName);
+									if (exc != null)
+									{
+										Lucene.Net.Index.SegmentInfos.Message("success on fallback " + prevSegmentFileName);
+									}
+									return v;
+								}
+								catch (System.IO.IOException err2)
+								{
+									Lucene.Net.Index.SegmentInfos.Message("secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry");
+								}
+							}
+						}
+					}
+				}
+			}
 			
-			SegmentInfos sis = new SegmentInfos();
-			sis.Read(directory);
-			return sis.GetVersion();
+			/// <summary> Subclass must implement this.  The assumption is an
+			/// IOException will be thrown if something goes wrong
+			/// during the processing that could have been caused by
+			/// a writer committing.
+			/// </summary>
+			public abstract System.Object DoBody(System.String segmentFileName);
 		}
 	}
 }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/SegmentMerger.cs?view=diff&rev=534192&r1=534191&r2=534192
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs Tue May  1 11:45:26 2007
@@ -16,6 +16,8 @@
  */
 
 using System;
+using FieldSelector = Lucene.Net.Documents.FieldSelector;
+using FieldSelectorResult = Lucene.Net.Documents.FieldSelectorResult;
 using Directory = Lucene.Net.Store.Directory;
 using IndexOutput = Lucene.Net.Store.IndexOutput;
 using RAMOutputStream = Lucene.Net.Store.RAMOutputStream;
@@ -31,16 +33,44 @@
 	/// 
 	/// 
 	/// </summary>
-	/// <seealso cref="merge">
+	/// <seealso cref="#merge">
 	/// </seealso>
-	/// <seealso cref="add">
+	/// <seealso cref="#add">
 	/// </seealso>
-	public sealed class SegmentMerger
+	sealed class SegmentMerger
 	{
+		private class AnonymousClassFieldSelector : FieldSelector
+		{
+			public AnonymousClassFieldSelector(SegmentMerger enclosingInstance)
+			{
+				InitBlock(enclosingInstance);
+			}
+			private void  InitBlock(SegmentMerger enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private SegmentMerger enclosingInstance;
+			public SegmentMerger Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			public FieldSelectorResult Accept(System.String fieldName)
+			{
+				return FieldSelectorResult.LOAD_FOR_MERGE;
+			}
+		}
 		private void  InitBlock()
 		{
 			termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL;
 		}
+		
+		/// <summary>norms header placeholder </summary>
+		internal static readonly byte[] NORMS_HEADER = new byte[]{(byte) 'N', (byte) 'R', (byte) 'M', (byte) 255};
+		
 		private Directory directory;
 		private System.String segment;
 		private int termIndexInterval;
@@ -55,7 +85,7 @@
 		/// </param>
 		/// <param name="name">The name of the new segment
 		/// </param>
-		public /*internal*/ SegmentMerger(Directory dir, System.String name)
+		public SegmentMerger(Directory dir, System.String name)
 		{
 			InitBlock();
 			directory = dir;
@@ -71,9 +101,9 @@
 		}
 		
 		/// <summary> Add an IndexReader to the collection of readers that are to be merged</summary>
-		/// <param name="reader">
+		/// <param name="">reader
 		/// </param>
-		public /*internal*/ void  Add(IndexReader reader)
+		public void  Add(IndexReader reader)
 		{
 			readers.Add(reader);
 		}
@@ -92,7 +122,7 @@
 		/// <returns> The number of documents that were merged
 		/// </returns>
 		/// <throws>  IOException </throws>
-		public /*internal*/ int Merge()
+		public int Merge()
 		{
 			int value_Renamed;
 			
@@ -110,7 +140,7 @@
 		/// Should not be called before merge().
 		/// </summary>
 		/// <throws>  IOException </throws>
-		public /*internal*/ void  CloseReaders()
+		public void  CloseReaders()
 		{
 			for (int i = 0; i < readers.Count; i++)
 			{
@@ -124,7 +154,7 @@
 		{
 			CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName);
 			
-			System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + fieldInfos.Size()));
+			System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(IndexFileNames.COMPOUND_EXTENSIONS.Length + 1));
 			
 			// Basic files
 			for (int i = 0; i < IndexFileNames.COMPOUND_EXTENSIONS.Length; i++)
@@ -132,13 +162,14 @@
 				files.Add(segment + "." + IndexFileNames.COMPOUND_EXTENSIONS[i]);
 			}
 			
-			// Field norm files
+			// Fieldable norm files
 			for (int i = 0; i < fieldInfos.Size(); i++)
 			{
 				FieldInfo fi = fieldInfos.FieldInfo(i);
 				if (fi.isIndexed && !fi.omitNorms)
 				{
-					files.Add(segment + ".f" + i);
+					files.Add(segment + "." + IndexFileNames.NORMS_EXTENSION);
+					break;
 				}
 			}
 			
@@ -169,8 +200,8 @@
 			System.Collections.IEnumerator i = names.GetEnumerator();
 			while (i.MoveNext())
 			{
-                System.Collections.DictionaryEntry e = (System.Collections.DictionaryEntry) i.Current;
-                System.String field = (System.String) e.Key;
+				System.Collections.DictionaryEntry e = (System.Collections.DictionaryEntry) i.Current;
+				System.String field = (System.String) e.Key;
 				fieldInfos.Add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.HasNorms(field));
 			}
 		}
@@ -196,6 +227,11 @@
 			fieldInfos.Write(directory, segment + ".fnm");
 			
 			FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos);
+			
+			// for merging we don't want to compress/uncompress the data, so to tell the FieldsReader that we're
+			// in  merge mode, we use this FieldSelector
+			FieldSelector fieldSelectorMerge = new AnonymousClassFieldSelector(this);
+			
 			try
 			{
 				for (int i = 0; i < readers.Count; i++)
@@ -206,7 +242,7 @@
 						if (!reader.IsDeleted(j))
 						{
 							// skip deleted docs
-							fieldsWriter.AddDocument(reader.Document(j));
+							fieldsWriter.AddDocument(reader.Document(j, fieldSelectorMerge));
 							docCount++;
 						}
 				}
@@ -379,8 +415,8 @@
 						doc = docMap[doc]; // map around deletions
 					doc += base_Renamed; // convert to merged space
 					
-					if (doc < lastDoc)
-						throw new System.SystemException("docs out of order (" + doc + " < " + lastDoc + " )");
+					if (doc < 0 || (df > 0 && doc <= lastDoc))
+						throw new System.SystemException("docs out of order (" + doc + " <= " + lastDoc + " )");
 					
 					df++;
 					
@@ -451,33 +487,56 @@
 		
 		private void  MergeNorms()
 		{
-			for (int i = 0; i < fieldInfos.Size(); i++)
+			byte[] normBuffer = null;
+			IndexOutput output = null;
+			try
 			{
-				FieldInfo fi = fieldInfos.FieldInfo(i);
-				if (fi.isIndexed && !fi.omitNorms)
+				for (int i = 0; i < fieldInfos.Size(); i++)
 				{
-					IndexOutput output = directory.CreateOutput(segment + ".f" + i);
-					try
+					FieldInfo fi = fieldInfos.FieldInfo(i);
+					if (fi.isIndexed && !fi.omitNorms)
 					{
+						if (output == null)
+						{
+							output = directory.CreateOutput(segment + "." + IndexFileNames.NORMS_EXTENSION);
+							output.WriteBytes(NORMS_HEADER, NORMS_HEADER.Length);
+						}
 						for (int j = 0; j < readers.Count; j++)
 						{
 							IndexReader reader = (IndexReader) readers[j];
 							int maxDoc = reader.MaxDoc();
-							byte[] input = new byte[maxDoc];
-							reader.Norms(fi.name, input, 0);
-							for (int k = 0; k < maxDoc; k++)
+							if (normBuffer == null || normBuffer.Length < maxDoc)
+							{
+								// the buffer is too small for the current segment
+								normBuffer = new byte[maxDoc];
+							}
+							reader.Norms(fi.name, normBuffer, 0);
+							if (!reader.HasDeletions())
 							{
-								if (!reader.IsDeleted(k))
+								//optimized case for segments without deleted docs
+								output.WriteBytes(normBuffer, maxDoc);
+							}
+							else
+							{
+								// this segment has deleted docs, so we have to
+								// check for every doc if it is deleted or not
+								for (int k = 0; k < maxDoc; k++)
 								{
-									output.WriteByte(input[k]);
+									if (!reader.IsDeleted(k))
+									{
+										output.WriteByte(normBuffer[k]);
+									}
 								}
 							}
 						}
 					}
-					finally
-					{
-						output.Close();
-					}
+				}
+			}
+			finally
+			{
+				if (output != null)
+				{
+					output.Close();
 				}
 			}
 		}



Mime
View raw message