lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ccurr...@apache.org
Subject [24/51] [partial] Mass convert mixed tabs to spaces
Date Wed, 03 Apr 2013 17:40:07 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Token.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Token.cs b/src/core/Analysis/Token.cs
index 3357f34..ec5e29b 100644
--- a/src/core/Analysis/Token.cs
+++ b/src/core/Analysis/Token.cs
@@ -26,780 +26,780 @@ using Attribute = Lucene.Net.Util.Attribute;
 
 namespace Lucene.Net.Analysis
 {
-	
-	/// <summary>A Token is an occurrence of a term from the text of a field.  It consists of
-	/// a term's text, the start and end offset of the term in the text of the field,
-	/// and a type string.
-	/// <p/>
-	/// The start and end offsets permit applications to re-associate a token with
-	/// its source text, e.g., to display highlighted query terms in a document
-	/// browser, or to show matching text fragments in a <abbr
-	/// title="KeyWord In Context">KWIC</abbr> display, etc.
-	/// <p/>
-	/// The type is a string, assigned by a lexical analyzer
-	/// (a.k.a. tokenizer), naming the lexical or syntactic class that the token
-	/// belongs to.  For example an end of sentence marker token might be implemented
-	/// with type "eos".  The default token type is "word".  
-	/// <p/>
-	/// A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
-	/// length byte array. Use <see cref="TermPositions.PayloadLength" /> and 
-	/// <see cref="TermPositions.GetPayload(byte[], int)" /> to retrieve the payloads from the index.
-	/// </summary>
-	/// <summary><br/><br/>
-	/// </summary>
-	/// <summary><p/><b>NOTE:</b> As of 2.9, Token implements all <see cref="IAttribute" /> interfaces
-	/// that are part of core Lucene and can be found in the <see cref="Lucene.Net.Analysis.Tokenattributes"/> namespace.
-	/// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
-	/// be used as convenience class that implements all <see cref="IAttribute" />s, which is especially useful
-	/// to easily switch from the old to the new TokenStream API.
-	/// <br/><br/>
-	/// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
-	/// possible for best performance, by implementing the
-	/// <see cref="TokenStream.IncrementToken()" /> API.
-	/// Failing that, to create a new Token you should first use
-	/// one of the constructors that starts with null text.  To load
-	/// the token from a char[] use <see cref="SetTermBuffer(char[], int, int)" />.
-	/// To load from a String use <see cref="SetTermBuffer(String)" /> or <see cref="SetTermBuffer(String, int, int)" />.
-	/// Alternatively you can get the Token's termBuffer by calling either <see cref="TermBuffer()" />,
-	/// if you know that your text is shorter than the capacity of the termBuffer
-	/// or <see cref="ResizeTermBuffer(int)" />, if there is any possibility
-	/// that you may need to grow the buffer. Fill in the characters of your term into this
+    
+    /// <summary>A Token is an occurrence of a term from the text of a field.  It consists of
+    /// a term's text, the start and end offset of the term in the text of the field,
+    /// and a type string.
+    /// <p/>
+    /// The start and end offsets permit applications to re-associate a token with
+    /// its source text, e.g., to display highlighted query terms in a document
+    /// browser, or to show matching text fragments in a <abbr
+    /// title="KeyWord In Context">KWIC</abbr> display, etc.
+    /// <p/>
+    /// The type is a string, assigned by a lexical analyzer
+    /// (a.k.a. tokenizer), naming the lexical or syntactic class that the token
+    /// belongs to.  For example an end of sentence marker token might be implemented
+    /// with type "eos".  The default token type is "word".  
+    /// <p/>
+    /// A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
+    /// length byte array. Use <see cref="TermPositions.PayloadLength" /> and 
+    /// <see cref="TermPositions.GetPayload(byte[], int)" /> to retrieve the payloads from the index.
+    /// </summary>
+    /// <summary><br/><br/>
+    /// </summary>
+    /// <summary><p/><b>NOTE:</b> As of 2.9, Token implements all <see cref="IAttribute" /> interfaces
+    /// that are part of core Lucene and can be found in the <see cref="Lucene.Net.Analysis.Tokenattributes"/> namespace.
+    /// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
+    /// be used as convenience class that implements all <see cref="IAttribute" />s, which is especially useful
+    /// to easily switch from the old to the new TokenStream API.
+    /// <br/><br/>
+    /// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
+    /// possible for best performance, by implementing the
+    /// <see cref="TokenStream.IncrementToken()" /> API.
+    /// Failing that, to create a new Token you should first use
+    /// one of the constructors that starts with null text.  To load
+    /// the token from a char[] use <see cref="SetTermBuffer(char[], int, int)" />.
+    /// To load from a String use <see cref="SetTermBuffer(String)" /> or <see cref="SetTermBuffer(String, int, int)" />.
+    /// Alternatively you can get the Token's termBuffer by calling either <see cref="TermBuffer()" />,
+    /// if you know that your text is shorter than the capacity of the termBuffer
+    /// or <see cref="ResizeTermBuffer(int)" />, if there is any possibility
+    /// that you may need to grow the buffer. Fill in the characters of your term into this
     /// buffer, with <see cref="string.ToCharArray(int, int)" /> if loading from a string,
-	/// or with <see cref="Array.Copy(Array, long, Array, long, long)" />, and finally call <see cref="SetTermLength(int)" /> to
-	/// set the length of the term text.  See <a target="_top"
-	/// href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
-	/// for details.<p/>
-	/// <p/>Typical Token reuse patterns:
-	/// <list type="bullet">
-	/// <item> Copying text from a string (type is reset to <see cref="DEFAULT_TYPE" /> if not
-	/// specified):<br/>
-	/// <code>
-	/// return reusableToken.reinit(string, startOffset, endOffset[, type]);
-	/// </code>
-	/// </item>
-	/// <item> Copying some text from a string (type is reset to <see cref="DEFAULT_TYPE" />
-	/// if not specified):<br/>
+    /// or with <see cref="Array.Copy(Array, long, Array, long, long)" />, and finally call <see cref="SetTermLength(int)" /> to
+    /// set the length of the term text.  See <a target="_top"
+    /// href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
+    /// for details.<p/>
+    /// <p/>Typical Token reuse patterns:
+    /// <list type="bullet">
+    /// <item> Copying text from a string (type is reset to <see cref="DEFAULT_TYPE" /> if not
+    /// specified):<br/>
     /// <code>
-	/// return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
+    /// return reusableToken.reinit(string, startOffset, endOffset[, type]);
     /// </code>
-	/// </item>
-	/// <item> Copying text from char[] buffer (type is reset to <see cref="DEFAULT_TYPE" />
-	/// if not specified):<br/>
+    /// </item>
+    /// <item> Copying some text from a string (type is reset to <see cref="DEFAULT_TYPE" />
+    /// if not specified):<br/>
     /// <code>
-	/// return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
+    /// return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
     /// </code>
-	/// </item>
-	/// <item> Copying some text from a char[] buffer (type is reset to
-	/// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+    /// </item>
+    /// <item> Copying text from char[] buffer (type is reset to <see cref="DEFAULT_TYPE" />
+    /// if not specified):<br/>
     /// <code>
-	/// return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
+    /// return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
     /// </code>
-	/// </item>
-	/// <item> Copying from one one Token to another (type is reset to
-	/// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+    /// </item>
+    /// <item> Copying some text from a char[] buffer (type is reset to
+    /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
     /// <code>
-	/// return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
+    /// return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
     /// </code>
-	/// </item>
-	/// </list>
-	/// A few things to note:
-	/// <list type="bullet">
-	/// <item>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</item>
-	/// <item>Because <c>TokenStreams</c> can be chained, one cannot assume that the <c>Token's</c> current type is correct.</item>
-	/// <item>The startOffset and endOffset represent the start and offset in the
-	/// source text, so be careful in adjusting them.</item>
-	/// <item>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</item>
-	/// </list>
-	/// <p/>
-	/// </summary>
-	/// <seealso cref="Lucene.Net.Index.Payload">
-	/// </seealso>
-	[Serializable]
-	public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
-	{
-		public const String DEFAULT_TYPE = "word";
+    /// </item>
+    /// <item> Copying from one one Token to another (type is reset to
+    /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+    /// <code>
+    /// return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
+    /// </code>
+    /// </item>
+    /// </list>
+    /// A few things to note:
+    /// <list type="bullet">
+    /// <item>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</item>
+    /// <item>Because <c>TokenStreams</c> can be chained, one cannot assume that the <c>Token's</c> current type is correct.</item>
+    /// <item>The startOffset and endOffset represent the start and offset in the
+    /// source text, so be careful in adjusting them.</item>
+    /// <item>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</item>
+    /// </list>
+    /// <p/>
+    /// </summary>
+    /// <seealso cref="Lucene.Net.Index.Payload">
+    /// </seealso>
+    [Serializable]
+    public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
+    {
+        public const String DEFAULT_TYPE = "word";
 
-		private const int MIN_BUFFER_SIZE = 10;
+        private const int MIN_BUFFER_SIZE = 10;
 
-		private char[] termBuffer;
-		private int termLength;
-		private int startOffset, endOffset;
-		private string type = DEFAULT_TYPE;
-		private int flags;
-		private Payload payload;
-		private int positionIncrement = 1;
-		
-		/// <summary>Constructs a Token will null text. </summary>
-		public Token()
-		{
-		}
-		
-		/// <summary>Constructs a Token with null text and start &amp; end
-		/// offsets.
-		/// </summary>
-		/// <param name="start">start offset in the source text</param>
-		/// <param name="end">end offset in the source text</param>
-		public Token(int start, int end)
-		{
-			startOffset = start;
-			endOffset = end;
-		}
-		
-		/// <summary>Constructs a Token with null text and start &amp; end
-		/// offsets plus the Token type.
-		/// </summary>
-		/// <param name="start">start offset in the source text</param>
-		/// <param name="end">end offset in the source text</param>
-		/// <param name="typ">the lexical type of this Token</param>
-		public Token(int start, int end, String typ)
-		{
-			startOffset = start;
-			endOffset = end;
-			type = typ;
-		}
-		
-		/// <summary> Constructs a Token with null text and start &amp; end
-		/// offsets plus flags. NOTE: flags is EXPERIMENTAL.
-		/// </summary>
-		/// <param name="start">start offset in the source text</param>
-		/// <param name="end">end offset in the source text</param>
-		/// <param name="flags">The bits to set for this token</param>
-		public Token(int start, int end, int flags)
-		{
-			startOffset = start;
-			endOffset = end;
-			this.flags = flags;
-		}
-		
-		/// <summary>Constructs a Token with the given term text, and start
-		/// &amp; end offsets.  The type defaults to "word."
-		/// <b>NOTE:</b> for better indexing speed you should
-		/// instead use the char[] termBuffer methods to set the
-		/// term text.
-		/// </summary>
-		/// <param name="text">term text</param>
-		/// <param name="start">start offset</param>
-		/// <param name="end">end offset</param>
-		public Token(String text, int start, int end)
-		{
-		    SetTermBuffer(text);
-			startOffset = start;
-			endOffset = end;
-		}
-		
-		/// <summary>Constructs a Token with the given text, start and end
-		/// offsets, &amp; type.  <b>NOTE:</b> for better indexing
-		/// speed you should instead use the char[] termBuffer
-		/// methods to set the term text.
-		/// </summary>
-		/// <param name="text">term text</param>
-		/// <param name="start">start offset</param>
-		/// <param name="end">end offset</param>
-		/// <param name="typ">token type</param>
-		public Token(System.String text, int start, int end, System.String typ)
-		{
-		    SetTermBuffer(text);
-			startOffset = start;
-			endOffset = end;
-			type = typ;
-		}
-		
-		/// <summary>  Constructs a Token with the given text, start and end
-		/// offsets, &amp; type.  <b>NOTE:</b> for better indexing
-		/// speed you should instead use the char[] termBuffer
-		/// methods to set the term text.
-		/// </summary>
-		/// <param name="text"></param>
-		/// <param name="start"></param>
-		/// <param name="end"></param>
-		/// <param name="flags">token type bits</param>
-		public Token(System.String text, int start, int end, int flags)
-		{
-		    SetTermBuffer(text);
-			startOffset = start;
-			endOffset = end;
-			this.flags = flags;
-		}
-		
-		/// <summary>  Constructs a Token with the given term buffer (offset
-		/// &amp; length), start and end
-		/// offsets
-		/// </summary>
-		/// <param name="startTermBuffer"></param>
-		/// <param name="termBufferOffset"></param>
-		/// <param name="termBufferLength"></param>
-		/// <param name="start"></param>
-		/// <param name="end"></param>
-		public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
-		{
-			SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
-			startOffset = start;
-			endOffset = end;
-		}
+        private char[] termBuffer;
+        private int termLength;
+        private int startOffset, endOffset;
+        private string type = DEFAULT_TYPE;
+        private int flags;
+        private Payload payload;
+        private int positionIncrement = 1;
+        
+        /// <summary>Constructs a Token will null text. </summary>
+        public Token()
+        {
+        }
+        
+        /// <summary>Constructs a Token with null text and start &amp; end
+        /// offsets.
+        /// </summary>
+        /// <param name="start">start offset in the source text</param>
+        /// <param name="end">end offset in the source text</param>
+        public Token(int start, int end)
+        {
+            startOffset = start;
+            endOffset = end;
+        }
+        
+        /// <summary>Constructs a Token with null text and start &amp; end
+        /// offsets plus the Token type.
+        /// </summary>
+        /// <param name="start">start offset in the source text</param>
+        /// <param name="end">end offset in the source text</param>
+        /// <param name="typ">the lexical type of this Token</param>
+        public Token(int start, int end, String typ)
+        {
+            startOffset = start;
+            endOffset = end;
+            type = typ;
+        }
+        
+        /// <summary> Constructs a Token with null text and start &amp; end
+        /// offsets plus flags. NOTE: flags is EXPERIMENTAL.
+        /// </summary>
+        /// <param name="start">start offset in the source text</param>
+        /// <param name="end">end offset in the source text</param>
+        /// <param name="flags">The bits to set for this token</param>
+        public Token(int start, int end, int flags)
+        {
+            startOffset = start;
+            endOffset = end;
+            this.flags = flags;
+        }
+        
+        /// <summary>Constructs a Token with the given term text, and start
+        /// &amp; end offsets.  The type defaults to "word."
+        /// <b>NOTE:</b> for better indexing speed you should
+        /// instead use the char[] termBuffer methods to set the
+        /// term text.
+        /// </summary>
+        /// <param name="text">term text</param>
+        /// <param name="start">start offset</param>
+        /// <param name="end">end offset</param>
+        public Token(String text, int start, int end)
+        {
+            SetTermBuffer(text);
+            startOffset = start;
+            endOffset = end;
+        }
+        
+        /// <summary>Constructs a Token with the given text, start and end
+        /// offsets, &amp; type.  <b>NOTE:</b> for better indexing
+        /// speed you should instead use the char[] termBuffer
+        /// methods to set the term text.
+        /// </summary>
+        /// <param name="text">term text</param>
+        /// <param name="start">start offset</param>
+        /// <param name="end">end offset</param>
+        /// <param name="typ">token type</param>
+        public Token(System.String text, int start, int end, System.String typ)
+        {
+            SetTermBuffer(text);
+            startOffset = start;
+            endOffset = end;
+            type = typ;
+        }
+        
+        /// <summary>  Constructs a Token with the given text, start and end
+        /// offsets, &amp; type.  <b>NOTE:</b> for better indexing
+        /// speed you should instead use the char[] termBuffer
+        /// methods to set the term text.
+        /// </summary>
+        /// <param name="text"></param>
+        /// <param name="start"></param>
+        /// <param name="end"></param>
+        /// <param name="flags">token type bits</param>
+        public Token(System.String text, int start, int end, int flags)
+        {
+            SetTermBuffer(text);
+            startOffset = start;
+            endOffset = end;
+            this.flags = flags;
+        }
+        
+        /// <summary>  Constructs a Token with the given term buffer (offset
+        /// &amp; length), start and end
+        /// offsets
+        /// </summary>
+        /// <param name="startTermBuffer"></param>
+        /// <param name="termBufferOffset"></param>
+        /// <param name="termBufferLength"></param>
+        /// <param name="start"></param>
+        /// <param name="end"></param>
+        public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
+        {
+            SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+            startOffset = start;
+            endOffset = end;
+        }
 
-	    /// <summary>Set the position increment.  This determines the position of this token
-	    /// relative to the previous Token in a <see cref="TokenStream" />, used in phrase
-	    /// searching.
-	    /// 
-	    /// <p/>The default value is one.
-	    /// 
-	    /// <p/>Some common uses for this are:<list>
-	    /// 
-	    /// <item>Set it to zero to put multiple terms in the same position.  This is
-	    /// useful if, e.g., a word has multiple stems.  Searches for phrases
-	    /// including either stem will match.  In this case, all but the first stem's
-	    /// increment should be set to zero: the increment of the first instance
-	    /// should be one.  Repeating a token with an increment of zero can also be
-	    /// used to boost the scores of matches on that token.</item>
-	    /// 
-	    /// <item>Set it to values greater than one to inhibit exact phrase matches.
-	    /// If, for example, one does not want phrases to match across removed stop
-	    /// words, then one could build a stop word filter that removes stop words and
-	    /// also sets the increment to the number of stop words removed before each
-	    /// non-stop word.  Then exact phrase queries will only match when the terms
-	    /// occur with no intervening stop words.</item>
-	    /// 
-	    /// </list>
-	    /// </summary>
-	    /// <value> the distance from the prior term </value>
-	    /// <seealso cref="Lucene.Net.Index.TermPositions">
-	    /// </seealso>
-	    public virtual int PositionIncrement
-	    {
-	        set
-	        {
-	            if (value < 0)
-	                throw new System.ArgumentException("Increment must be zero or greater: " + value);
-	            this.positionIncrement = value;
-	        }
-	        get { return positionIncrement; }
-	    }
+        /// <summary>Set the position increment.  This determines the position of this token
+        /// relative to the previous Token in a <see cref="TokenStream" />, used in phrase
+        /// searching.
+        /// 
+        /// <p/>The default value is one.
+        /// 
+        /// <p/>Some common uses for this are:<list>
+        /// 
+        /// <item>Set it to zero to put multiple terms in the same position.  This is
+        /// useful if, e.g., a word has multiple stems.  Searches for phrases
+        /// including either stem will match.  In this case, all but the first stem's
+        /// increment should be set to zero: the increment of the first instance
+        /// should be one.  Repeating a token with an increment of zero can also be
+        /// used to boost the scores of matches on that token.</item>
+        /// 
+        /// <item>Set it to values greater than one to inhibit exact phrase matches.
+        /// If, for example, one does not want phrases to match across removed stop
+        /// words, then one could build a stop word filter that removes stop words and
+        /// also sets the increment to the number of stop words removed before each
+        /// non-stop word.  Then exact phrase queries will only match when the terms
+        /// occur with no intervening stop words.</item>
+        /// 
+        /// </list>
+        /// </summary>
+        /// <value> the distance from the prior term </value>
+        /// <seealso cref="Lucene.Net.Index.TermPositions">
+        /// </seealso>
+        public virtual int PositionIncrement
+        {
+            set
+            {
+                if (value < 0)
+                    throw new System.ArgumentException("Increment must be zero or greater: " + value);
+                this.positionIncrement = value;
+            }
+            get { return positionIncrement; }
+        }
 
-	    /// <summary>Returns the Token's term text.
-	    /// 
-	    /// This method has a performance penalty
-	    /// because the text is stored internally in a char[].  If
-	    /// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
-	    /// directly instead.  If you really need a
-	    /// String, use this method, which is nothing more than
-	    /// a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
-	    /// </summary>
-	    public string Term
-	    {
-	        get
-	        {
-	            InitTermBuffer();
-	            return new System.String(termBuffer, 0, termLength);
-	        }
-	    }
+        /// <summary>Returns the Token's term text.
+        /// 
+        /// This method has a performance penalty
+        /// because the text is stored internally in a char[].  If
+        /// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
+        /// directly instead.  If you really need a
+        /// String, use this method, which is nothing more than
+        /// a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
+        /// </summary>
+        public string Term
+        {
+            get
+            {
+                InitTermBuffer();
+                return new System.String(termBuffer, 0, termLength);
+            }
+        }
 
-	    /// <summary>Copies the contents of buffer, starting at offset for
-		/// length characters, into the termBuffer array.
-		/// </summary>
-		/// <param name="buffer">the buffer to copy</param>
-		/// <param name="offset">the index in the buffer of the first character to copy</param>
-		/// <param name="length">the number of characters to copy</param>
-		public void  SetTermBuffer(char[] buffer, int offset, int length)
-		{
-			GrowTermBuffer(length);
-			Array.Copy(buffer, offset, termBuffer, 0, length);
-			termLength = length;
-		}
-		
-		/// <summary>Copies the contents of buffer into the termBuffer array.</summary>
-		/// <param name="buffer">the buffer to copy
-		/// </param>
-		public void  SetTermBuffer(System.String buffer)
-		{
-			int length = buffer.Length;
-			GrowTermBuffer(length);
-			TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
-			termLength = length;
-		}
-		
-		/// <summary>Copies the contents of buffer, starting at offset and continuing
-		/// for length characters, into the termBuffer array.
-		/// </summary>
-		/// <param name="buffer">the buffer to copy
-		/// </param>
-		/// <param name="offset">the index in the buffer of the first character to copy
-		/// </param>
-		/// <param name="length">the number of characters to copy
-		/// </param>
-		public void  SetTermBuffer(System.String buffer, int offset, int length)
-		{
-			System.Diagnostics.Debug.Assert(offset <= buffer.Length);
-			System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
-			GrowTermBuffer(length);
-			TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
-			termLength = length;
-		}
-		
-		/// <summary>Returns the internal termBuffer character array which
-		/// you can then directly alter.  If the array is too
-		/// small for your token, use <see cref="ResizeTermBuffer(int)" />
-		/// to increase it.  After
-		/// altering the buffer be sure to call <see cref="SetTermLength" />
-		/// to record the number of valid
-		/// characters that were placed into the termBuffer. 
-		/// </summary>
-		public char[] TermBuffer()
-		{
-			InitTermBuffer();
-			return termBuffer;
-		}
-		
-		/// <summary>Grows the termBuffer to at least size newSize, preserving the
-		/// existing content. Note: If the next operation is to change
-		/// the contents of the term buffer use
-		/// <see cref="SetTermBuffer(char[], int, int)" />,
-		/// <see cref="SetTermBuffer(String)" />, or
-		/// <see cref="SetTermBuffer(String, int, int)" />
-		/// to optimally combine the resize with the setting of the termBuffer.
-		/// </summary>
-		/// <param name="newSize">minimum size of the new termBuffer
-		/// </param>
-		/// <returns> newly created termBuffer with length >= newSize
-		/// </returns>
-		public virtual char[] ResizeTermBuffer(int newSize)
-		{
-			if (termBuffer == null)
-			{
+        /// <summary>Copies the contents of buffer, starting at offset for
+        /// length characters, into the termBuffer array.
+        /// </summary>
+        /// <param name="buffer">the buffer to copy</param>
+        /// <param name="offset">the index in the buffer of the first character to copy</param>
+        /// <param name="length">the number of characters to copy</param>
+        public void  SetTermBuffer(char[] buffer, int offset, int length)
+        {
+            GrowTermBuffer(length);
+            Array.Copy(buffer, offset, termBuffer, 0, length);
+            termLength = length;
+        }
+        
+        /// <summary>Copies the contents of buffer into the termBuffer array.</summary>
+        /// <param name="buffer">the buffer to copy
+        /// </param>
+        public void  SetTermBuffer(System.String buffer)
+        {
+            int length = buffer.Length;
+            GrowTermBuffer(length);
+            TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
+            termLength = length;
+        }
+        
+        /// <summary>Copies the contents of buffer, starting at offset and continuing
+        /// for length characters, into the termBuffer array.
+        /// </summary>
+        /// <param name="buffer">the buffer to copy
+        /// </param>
+        /// <param name="offset">the index in the buffer of the first character to copy
+        /// </param>
+        /// <param name="length">the number of characters to copy
+        /// </param>
+        public void  SetTermBuffer(System.String buffer, int offset, int length)
+        {
+            System.Diagnostics.Debug.Assert(offset <= buffer.Length);
+            System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
+            GrowTermBuffer(length);
+            TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
+            termLength = length;
+        }
+        
+        /// <summary>Returns the internal termBuffer character array which
+        /// you can then directly alter.  If the array is too
+        /// small for your token, use <see cref="ResizeTermBuffer(int)" />
+        /// to increase it.  After
+        /// altering the buffer be sure to call <see cref="SetTermLength" />
+        /// to record the number of valid
+        /// characters that were placed into the termBuffer. 
+        /// </summary>
+        public char[] TermBuffer()
+        {
+            InitTermBuffer();
+            return termBuffer;
+        }
+        
+        /// <summary>Grows the termBuffer to at least size newSize, preserving the
+        /// existing content. Note: If the next operation is to change
+        /// the contents of the term buffer use
+        /// <see cref="SetTermBuffer(char[], int, int)" />,
+        /// <see cref="SetTermBuffer(String)" />, or
+        /// <see cref="SetTermBuffer(String, int, int)" />
+        /// to optimally combine the resize with the setting of the termBuffer.
+        /// </summary>
+        /// <param name="newSize">minimum size of the new termBuffer
+        /// </param>
+        /// <returns> newly created termBuffer with length >= newSize
+        /// </returns>
+        public virtual char[] ResizeTermBuffer(int newSize)
+        {
+            if (termBuffer == null)
+            {
                 termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
-			}
-			else
-			{
-				if (termBuffer.Length < newSize)
-				{
-					// Not big enough; create a new array with slight
-					// over allocation and preserve content
-					var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
-					Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
-					termBuffer = newCharBuffer;
-				}
-			}
-			return termBuffer;
-		}
-		
-		/// <summary>Allocates a buffer char[] of at least newSize, without preserving the existing content.
-		/// its always used in places that set the content 
-		/// </summary>
-		/// <param name="newSize">minimum size of the buffer
-		/// </param>
-		private void  GrowTermBuffer(int newSize)
-		{
-			if (termBuffer == null)
-			{
-				// The buffer is always at least MIN_BUFFER_SIZE    
-				termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
-			}
-			else
-			{
-				if (termBuffer.Length < newSize)
-				{
-					// Not big enough; create a new array with slight
-					// over allocation:
-					termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
-				}
-			}
-		}
-		
-		private void  InitTermBuffer()
-		{
-			if (termBuffer == null)
-			{
+            }
+            else
+            {
+                if (termBuffer.Length < newSize)
+                {
+                    // Not big enough; create a new array with slight
+                    // over allocation and preserve content
+                    var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
+                    Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
+                    termBuffer = newCharBuffer;
+                }
+            }
+            return termBuffer;
+        }
+        
+        /// <summary>Allocates a buffer char[] of at least newSize, without preserving the existing content.
+        /// its always used in places that set the content 
+        /// </summary>
+        /// <param name="newSize">minimum size of the buffer
+        /// </param>
+        private void  GrowTermBuffer(int newSize)
+        {
+            if (termBuffer == null)
+            {
+                // The buffer is always at least MIN_BUFFER_SIZE    
+                termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
+            }
+            else
+            {
+                if (termBuffer.Length < newSize)
+                {
+                    // Not big enough; create a new array with slight
+                    // over allocation:
+                    termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
+                }
+            }
+        }
+        
+        private void  InitTermBuffer()
+        {
+            if (termBuffer == null)
+            {
                 termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
                 termLength = 0;
-			}
-		}
-		
-		/// <summary>Return number of valid characters (length of the term)
-		/// in the termBuffer array. 
-		/// </summary>
-		public int TermLength()
-		{
-			InitTermBuffer();
-			return termLength;
-		}
-		
-		/// <summary>Set number of valid characters (length of the term) in
-		/// the termBuffer array. Use this to truncate the termBuffer
-		/// or to synchronize with external manipulation of the termBuffer.
-		/// Note: to grow the size of the array,
-		/// use <see cref="ResizeTermBuffer(int)" /> first.
-		/// </summary>
-		/// <param name="length">the truncated length
-		/// </param>
-		public void  SetTermLength(int length)
-		{
-			InitTermBuffer();
-			if (length > termBuffer.Length)
-				throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
-			termLength = length;
-		}
+            }
+        }
+        
+        /// <summary>Return number of valid characters (length of the term)
+        /// in the termBuffer array. 
+        /// </summary>
+        public int TermLength()
+        {
+            InitTermBuffer();
+            return termLength;
+        }
+        
+        /// <summary>Set number of valid characters (length of the term) in
+        /// the termBuffer array. Use this to truncate the termBuffer
+        /// or to synchronize with external manipulation of the termBuffer.
+        /// Note: to grow the size of the array,
+        /// use <see cref="ResizeTermBuffer(int)" /> first.
+        /// </summary>
+        /// <param name="length">the truncated length
+        /// </param>
+        public void  SetTermLength(int length)
+        {
+            InitTermBuffer();
+            if (length > termBuffer.Length)
+                throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
+            termLength = length;
+        }
 
-	    /// <summary>Gets or sets this Token's starting offset, the position of the first character
-	    /// corresponding to this token in the source text.
-	    /// Note that the difference between endOffset() and startOffset() may not be
-	    /// equal to <see cref="TermLength"/>, as the term text may have been altered by a
-	    /// stemmer or some other filter. 
-	    /// </summary>
-	    public virtual int StartOffset
-	    {
-	        get { return startOffset; }
-	        set { this.startOffset = value; }
-	    }
+        /// <summary>Gets or sets this Token's starting offset, the position of the first character
+        /// corresponding to this token in the source text.
+        /// Note that the difference between endOffset() and startOffset() may not be
+        /// equal to <see cref="TermLength"/>, as the term text may have been altered by a
+        /// stemmer or some other filter. 
+        /// </summary>
+        public virtual int StartOffset
+        {
+            get { return startOffset; }
+            set { this.startOffset = value; }
+        }
 
-	    /// <summary>Gets or sets this Token's ending offset, one greater than the position of the
-	    /// last character corresponding to this token in the source text. The length
-	    /// of the token in the source text is (endOffset - startOffset). 
-	    /// </summary>
-	    public virtual int EndOffset
-	    {
-	        get { return endOffset; }
-	        set { this.endOffset = value; }
-	    }
+        /// <summary>Gets or sets this Token's ending offset, one greater than the position of the
+        /// last character corresponding to this token in the source text. The length
+        /// of the token in the source text is (endOffset - startOffset). 
+        /// </summary>
+        public virtual int EndOffset
+        {
+            get { return endOffset; }
+            set { this.endOffset = value; }
+        }
 
-	    /// <summary>Set the starting and ending offset.
-		/// See StartOffset() and EndOffset()
-		/// </summary>
-		public virtual void  SetOffset(int startOffset, int endOffset)
-		{
-			this.startOffset = startOffset;
-			this.endOffset = endOffset;
-		}
+        /// <summary>Set the starting and ending offset.
+        /// See StartOffset() and EndOffset()
+        /// </summary>
+        public virtual void  SetOffset(int startOffset, int endOffset)
+        {
+            this.startOffset = startOffset;
+            this.endOffset = endOffset;
+        }
 
-	    /// <summary>Returns this Token's lexical type.  Defaults to "word". </summary>
-	    public string Type
-	    {
-	        get { return type; }
-	        set { this.type = value; }
-	    }
+        /// <summary>Returns this Token's lexical type.  Defaults to "word". </summary>
+        public string Type
+        {
+            get { return type; }
+            set { this.type = value; }
+        }
 
-	    /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
-	    /// <p/>
-	    /// 
-	    /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="Type()" />, although they do share similar purposes.
-	    /// The flags can be used to encode information about the token for use by other <see cref="TokenFilter"/>s.
-	    /// 
-	    /// 
-	    /// </summary>
-	    /// <value> The bits </value>
-	    public virtual int Flags
-	    {
-	        get { return flags; }
-	        set { flags = value; }
-	    }
+        /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
+        /// <p/>
+        /// 
+        /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="Type()" />, although they do share similar purposes.
+        /// The flags can be used to encode information about the token for use by other <see cref="TokenFilter"/>s.
+        /// 
+        /// 
+        /// </summary>
+        /// <value> The bits </value>
+        public virtual int Flags
+        {
+            get { return flags; }
+            set { flags = value; }
+        }
 
-	    /// <summary> Returns this Token's payload.</summary>
-	    public virtual Payload Payload
-	    {
-	        get { return payload; }
-	        set { payload = value; }
-	    }
+        /// <summary> Returns this Token's payload.</summary>
+        public virtual Payload Payload
+        {
+            get { return payload; }
+            set { payload = value; }
+        }
 
-	    public override String ToString()
-		{
-			var sb = new System.Text.StringBuilder();
-			sb.Append('(');
-			InitTermBuffer();
-			if (termBuffer == null)
-				sb.Append("null");
-			else
-				sb.Append(termBuffer, 0, termLength);
-			sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
-			if (!type.Equals("word"))
-				sb.Append(",type=").Append(type);
-			if (positionIncrement != 1)
-				sb.Append(",posIncr=").Append(positionIncrement);
-			sb.Append(')');
-			return sb.ToString();
-		}
-		
-		/// <summary>Resets the term text, payload, flags, and positionIncrement,
-		/// startOffset, endOffset and token type to default.
-		/// </summary>
-		public override void  Clear()
-		{
-			payload = null;
-			// Leave termBuffer to allow re-use
-			termLength = 0;
-			positionIncrement = 1;
-			flags = 0;
-			startOffset = endOffset = 0;
-			type = DEFAULT_TYPE;
-		}
-		
-		public override System.Object Clone()
-		{
-			var t = (Token) base.Clone();
-			// Do a deep clone
-			if (termBuffer != null)
-			{
-				t.termBuffer = new char[termBuffer.Length];
-				termBuffer.CopyTo(t.termBuffer, 0);
-			}
-			if (payload != null)
-			{
-				t.payload = (Payload) payload.Clone();
-			}
-			return t;
-		}
-		
-		/// <summary>Makes a clone, but replaces the term buffer &amp;
-		/// start/end offset in the process.  This is more
-		/// efficient than doing a full clone (and then calling
-		/// setTermBuffer) because it saves a wasted copy of the old
-		/// termBuffer. 
-		/// </summary>
-		public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
-		{
-			var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
-			        	{positionIncrement = positionIncrement, flags = flags, type = type};
-			if (payload != null)
-				t.payload = (Payload) payload.Clone();
-			return t;
-		}
-		
-		public  override bool Equals(Object obj)
-		{
-			if (obj == this)
-				return true;
+        public override String ToString()
+        {
+            var sb = new System.Text.StringBuilder();
+            sb.Append('(');
+            InitTermBuffer();
+            if (termBuffer == null)
+                sb.Append("null");
+            else
+                sb.Append(termBuffer, 0, termLength);
+            sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
+            if (!type.Equals("word"))
+                sb.Append(",type=").Append(type);
+            if (positionIncrement != 1)
+                sb.Append(",posIncr=").Append(positionIncrement);
+            sb.Append(')');
+            return sb.ToString();
+        }
+        
+        /// <summary>Resets the term text, payload, flags, and positionIncrement,
+        /// startOffset, endOffset and token type to default.
+        /// </summary>
+        public override void  Clear()
+        {
+            payload = null;
+            // Leave termBuffer to allow re-use
+            termLength = 0;
+            positionIncrement = 1;
+            flags = 0;
+            startOffset = endOffset = 0;
+            type = DEFAULT_TYPE;
+        }
+        
+        public override System.Object Clone()
+        {
+            var t = (Token) base.Clone();
+            // Do a deep clone
+            if (termBuffer != null)
+            {
+                t.termBuffer = new char[termBuffer.Length];
+                termBuffer.CopyTo(t.termBuffer, 0);
+            }
+            if (payload != null)
+            {
+                t.payload = (Payload) payload.Clone();
+            }
+            return t;
+        }
+        
+        /// <summary>Makes a clone, but replaces the term buffer &amp;
+        /// start/end offset in the process.  This is more
+        /// efficient than doing a full clone (and then calling
+        /// setTermBuffer) because it saves a wasted copy of the old
+        /// termBuffer. 
+        /// </summary>
+        public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+        {
+            var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
+                        {positionIncrement = positionIncrement, flags = flags, type = type};
+            if (payload != null)
+                t.payload = (Payload) payload.Clone();
+            return t;
+        }
+        
+        public  override bool Equals(Object obj)
+        {
+            if (obj == this)
+                return true;
 
-			var other = obj as Token;
-			if (other == null)
-				return false;
-			
-			InitTermBuffer();
-			other.InitTermBuffer();
+            var other = obj as Token;
+            if (other == null)
+                return false;
+            
+            InitTermBuffer();
+            other.InitTermBuffer();
 
-			if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
-			    flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
-			    SubEqual(payload, other.payload))
-			{
-				for (int i = 0; i < termLength; i++)
-					if (termBuffer[i] != other.termBuffer[i])
-						return false;
-				return true;
-			}
-			return false;
-		}
-		
-		private bool SubEqual(System.Object o1, System.Object o2)
-		{
-			if (o1 == null)
-				return o2 == null;
-			return o1.Equals(o2);
-		}
+            if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
+                flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
+                SubEqual(payload, other.payload))
+            {
+                for (int i = 0; i < termLength; i++)
+                    if (termBuffer[i] != other.termBuffer[i])
+                        return false;
+                return true;
+            }
+            return false;
+        }
+        
+        private bool SubEqual(System.Object o1, System.Object o2)
+        {
+            if (o1 == null)
+                return o2 == null;
+            return o1.Equals(o2);
+        }
 
-		public override int GetHashCode()
-		{
-			InitTermBuffer();
-			int code = termLength;
-			code = code * 31 + startOffset;
-			code = code * 31 + endOffset;
-			code = code * 31 + flags;
-			code = code * 31 + positionIncrement;
-			code = code * 31 + type.GetHashCode();
-			code = (payload == null?code:code * 31 + payload.GetHashCode());
-			code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
-			return code;
-		}
-		
-		// like clear() but doesn't clear termBuffer/text
-		private void  ClearNoTermBuffer()
-		{
-			payload = null;
-			positionIncrement = 1;
-			flags = 0;
-			startOffset = endOffset = 0;
-			type = DEFAULT_TYPE;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(char[], int, int)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />,
-		/// <see cref="Type" />
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
-		{
-			ClearNoTermBuffer();
-			payload = null;
-			positionIncrement = 1;
-			SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = newType;
-			return this;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(char[], int, int)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />
-		/// <see cref="Type" /> on Token.DEFAULT_TYPE
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
-		{
-			ClearNoTermBuffer();
-			SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = DEFAULT_TYPE;
-			return this;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(String)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />
-		/// <see cref="Type" />
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
-		{
-			ClearNoTermBuffer();
-			SetTermBuffer(newTerm);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = newType;
-			return this;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(String, int, int)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />
-		/// <see cref="Type" />
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
-		{
-			ClearNoTermBuffer();
-			SetTermBuffer(newTerm, newTermOffset, newTermLength);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = newType;
-			return this;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(String)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />
-		/// <see cref="Type" /> on Token.DEFAULT_TYPE
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
-		{
-			ClearNoTermBuffer();
-			SetTermBuffer(newTerm);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = DEFAULT_TYPE;
-			return this;
-		}
-		
-		/// <summary>Shorthand for calling <see cref="Clear" />,
-		/// <see cref="SetTermBuffer(String, int, int)" />,
-		/// <see cref="StartOffset" />,
-		/// <see cref="EndOffset" />
-		/// <see cref="Type" /> on Token.DEFAULT_TYPE
-		/// </summary>
-		/// <returns> this Token instance 
-		/// </returns>
-		public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
-		{
-			ClearNoTermBuffer();
-			SetTermBuffer(newTerm, newTermOffset, newTermLength);
-			startOffset = newStartOffset;
-			endOffset = newEndOffset;
-			type = DEFAULT_TYPE;
-			return this;
-		}
-		
-		/// <summary> Copy the prototype token's fields into this one. Note: Payloads are shared.</summary>
-		/// <param name="prototype">
-		/// </param>
-		public virtual void  Reinit(Token prototype)
-		{
-			prototype.InitTermBuffer();
-			SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
-			positionIncrement = prototype.positionIncrement;
-			flags = prototype.flags;
-			startOffset = prototype.startOffset;
-			endOffset = prototype.endOffset;
-			type = prototype.type;
-			payload = prototype.payload;
-		}
-		
-		/// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
-		/// <param name="prototype">
-		/// </param>
-		/// <param name="newTerm">
-		/// </param>
-		public virtual void  Reinit(Token prototype, System.String newTerm)
-		{
-			SetTermBuffer(newTerm);
-			positionIncrement = prototype.positionIncrement;
-			flags = prototype.flags;
-			startOffset = prototype.startOffset;
-			endOffset = prototype.endOffset;
-			type = prototype.type;
-			payload = prototype.payload;
-		}
-		
-		/// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
-		/// <param name="prototype">
-		/// </param>
-		/// <param name="newTermBuffer">
-		/// </param>
-		/// <param name="offset">
-		/// </param>
-		/// <param name="length">
-		/// </param>
-		public virtual void  Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
-		{
-			SetTermBuffer(newTermBuffer, offset, length);
-			positionIncrement = prototype.positionIncrement;
-			flags = prototype.flags;
-			startOffset = prototype.startOffset;
-			endOffset = prototype.endOffset;
-			type = prototype.type;
-			payload = prototype.payload;
-		}
-		
-		public override void  CopyTo(Attribute target)
-		{
-			if (target is Token)
-			{
-				var to = (Token) target;
-				to.Reinit(this);
-				// reinit shares the payload, so clone it:
-				if (payload != null)
-				{
-					to.payload = (Payload) payload.Clone();
-				}
-			}
-			else
-			{
-				InitTermBuffer();
-				((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
-				((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
-				((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
-				((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
-				((IFlagsAttribute) target).Flags = flags;
-				((ITypeAttribute) target).Type = type;
-			}
-		}
+        public override int GetHashCode()
+        {
+            InitTermBuffer();
+            int code = termLength;
+            code = code * 31 + startOffset;
+            code = code * 31 + endOffset;
+            code = code * 31 + flags;
+            code = code * 31 + positionIncrement;
+            code = code * 31 + type.GetHashCode();
+            code = (payload == null?code:code * 31 + payload.GetHashCode());
+            code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
+            return code;
+        }
+        
+        // like clear() but doesn't clear termBuffer/text
+        private void  ClearNoTermBuffer()
+        {
+            payload = null;
+            positionIncrement = 1;
+            flags = 0;
+            startOffset = endOffset = 0;
+            type = DEFAULT_TYPE;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(char[], int, int)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />,
+        /// <see cref="Type" />
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
+        {
+            ClearNoTermBuffer();
+            payload = null;
+            positionIncrement = 1;
+            SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = newType;
+            return this;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(char[], int, int)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />
+        /// <see cref="Type" /> on Token.DEFAULT_TYPE
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+        {
+            ClearNoTermBuffer();
+            SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = DEFAULT_TYPE;
+            return this;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(String)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />
+        /// <see cref="Type" />
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
+        {
+            ClearNoTermBuffer();
+            SetTermBuffer(newTerm);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = newType;
+            return this;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(String, int, int)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />
+        /// <see cref="Type" />
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
+        {
+            ClearNoTermBuffer();
+            SetTermBuffer(newTerm, newTermOffset, newTermLength);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = newType;
+            return this;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(String)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />
+        /// <see cref="Type" /> on Token.DEFAULT_TYPE
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
+        {
+            ClearNoTermBuffer();
+            SetTermBuffer(newTerm);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = DEFAULT_TYPE;
+            return this;
+        }
+        
+        /// <summary>Shorthand for calling <see cref="Clear" />,
+        /// <see cref="SetTermBuffer(String, int, int)" />,
+        /// <see cref="StartOffset" />,
+        /// <see cref="EndOffset" />
+        /// <see cref="Type" /> on Token.DEFAULT_TYPE
+        /// </summary>
+        /// <returns> this Token instance 
+        /// </returns>
+        public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+        {
+            ClearNoTermBuffer();
+            SetTermBuffer(newTerm, newTermOffset, newTermLength);
+            startOffset = newStartOffset;
+            endOffset = newEndOffset;
+            type = DEFAULT_TYPE;
+            return this;
+        }
+        
+        /// <summary> Copy the prototype token's fields into this one. Note: Payloads are shared.</summary>
+        /// <param name="prototype">
+        /// </param>
+        public virtual void  Reinit(Token prototype)
+        {
+            prototype.InitTermBuffer();
+            SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
+            positionIncrement = prototype.positionIncrement;
+            flags = prototype.flags;
+            startOffset = prototype.startOffset;
+            endOffset = prototype.endOffset;
+            type = prototype.type;
+            payload = prototype.payload;
+        }
+        
+        /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
+        /// <param name="prototype">
+        /// </param>
+        /// <param name="newTerm">
+        /// </param>
+        public virtual void  Reinit(Token prototype, System.String newTerm)
+        {
+            SetTermBuffer(newTerm);
+            positionIncrement = prototype.positionIncrement;
+            flags = prototype.flags;
+            startOffset = prototype.startOffset;
+            endOffset = prototype.endOffset;
+            type = prototype.type;
+            payload = prototype.payload;
+        }
+        
+        /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
+        /// <param name="prototype">
+        /// </param>
+        /// <param name="newTermBuffer">
+        /// </param>
+        /// <param name="offset">
+        /// </param>
+        /// <param name="length">
+        /// </param>
+        public virtual void  Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
+        {
+            SetTermBuffer(newTermBuffer, offset, length);
+            positionIncrement = prototype.positionIncrement;
+            flags = prototype.flags;
+            startOffset = prototype.startOffset;
+            endOffset = prototype.endOffset;
+            type = prototype.type;
+            payload = prototype.payload;
+        }
+        
+        public override void  CopyTo(Attribute target)
+        {
+            if (target is Token)
+            {
+                var to = (Token) target;
+                to.Reinit(this);
+                // reinit shares the payload, so clone it:
+                if (payload != null)
+                {
+                    to.payload = (Payload) payload.Clone();
+                }
+            }
+            else
+            {
+                InitTermBuffer();
+                ((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
+                ((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
+                ((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
+                ((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
+                ((IFlagsAttribute) target).Flags = flags;
+                ((ITypeAttribute) target).Type = type;
+            }
+        }
        
         ///<summary>
         /// Convenience factory that returns <c>Token</c> as implementation for the basic
@@ -807,8 +807,8 @@ namespace Lucene.Net.Analysis
         /// attributes.
         /// @since 3.0
         /// </summary>
-	    public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
-	        new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+        public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+            new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
   
         /// <summary>
         /// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
@@ -839,8 +839,8 @@ namespace Lucene.Net.Analysis
             {
                 if (this == other) return true;
 
-            	var af = other as TokenAttributeFactory;
-            	return af != null && _delegateFactory.Equals(af._delegateFactory);
+                var af = other as TokenAttributeFactory;
+                return af != null && _delegateFactory.Equals(af._delegateFactory);
             }
 
             public override int GetHashCode()

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/TokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/TokenFilter.cs b/src/core/Analysis/TokenFilter.cs
index 7483c82..2cd1093 100644
--- a/src/core/Analysis/TokenFilter.cs
+++ b/src/core/Analysis/TokenFilter.cs
@@ -17,35 +17,35 @@
 
 namespace Lucene.Net.Analysis
 {
-	
-	/// <summary> A TokenFilter is a TokenStream whose input is another TokenStream.
-	/// <p/>
+    
+    /// <summary> A TokenFilter is a TokenStream whose input is another TokenStream.
+    /// <p/>
     /// This is an abstract class; subclasses must override <see cref="TokenStream.IncrementToken()" />.
-	/// 
-	/// </summary>
-	/// <seealso cref="TokenStream">
-	/// </seealso>
-	public abstract class TokenFilter:TokenStream
-	{
-		/// <summary>The source of tokens for this filter. </summary>
-		protected internal TokenStream input;
+    /// 
+    /// </summary>
+    /// <seealso cref="TokenStream">
+    /// </seealso>
+    public abstract class TokenFilter:TokenStream
+    {
+        /// <summary>The source of tokens for this filter. </summary>
+        protected internal TokenStream input;
 
-	    private bool isDisposed;
+        private bool isDisposed;
 
-	    /// <summary>Construct a token stream filtering the given input. </summary>
-		protected internal TokenFilter(TokenStream input):base(input)
-		{
-			this.input = input;
-		}
-		
-		/// <summary>Performs end-of-stream operations, if any, and calls then <c>end()</c> on the
-		/// input TokenStream.<p/> 
-		/// <b>NOTE:</b> Be sure to call <c>super.end()</c> first when overriding this method.
-		/// </summary>
-		public override void  End()
-		{
-			input.End();
-		}
+        /// <summary>Construct a token stream filtering the given input. </summary>
+        protected internal TokenFilter(TokenStream input):base(input)
+        {
+            this.input = input;
+        }
+        
+        /// <summary>Performs end-of-stream operations, if any, and calls then <c>end()</c> on the
+        /// input TokenStream.<p/> 
+        /// <b>NOTE:</b> Be sure to call <c>super.end()</c> first when overriding this method.
+        /// </summary>
+        public override void  End()
+        {
+            input.End();
+        }
 
         protected override void Dispose(bool disposing)
         {
@@ -62,11 +62,11 @@ namespace Lucene.Net.Analysis
             //input = null;
             isDisposed = true;
         }
-		
-		/// <summary>Reset the filter as well as the input TokenStream. </summary>
-		public override void  Reset()
-		{
-			input.Reset();
-		}
-	}
+        
+        /// <summary>Reset the filter as well as the input TokenStream. </summary>
+        public override void  Reset()
+        {
+            input.Reset();
+        }
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/TokenStream.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/TokenStream.cs b/src/core/Analysis/TokenStream.cs
index c624696..4a44609 100644
--- a/src/core/Analysis/TokenStream.cs
+++ b/src/core/Analysis/TokenStream.cs
@@ -24,139 +24,139 @@ using AttributeSource = Lucene.Net.Util.AttributeSource;
 
 namespace Lucene.Net.Analysis
 {
-	
-	/// <summary> A <c>TokenStream</c> enumerates the sequence of tokens, either from
-	/// <see cref="Field" />s of a <see cref="Document" /> or from query text.
-	/// <p/>
-	/// This is an abstract class. Concrete subclasses are:
-	/// <list type="bullet">
-	/// <item><see cref="Tokenizer" />, a <c>TokenStream</c> whose input is a Reader; and</item>
-	/// <item><see cref="TokenFilter" />, a <c>TokenStream</c> whose input is another
-	/// <c>TokenStream</c>.</item>
-	/// </list>
-	/// A new <c>TokenStream</c> API has been introduced with Lucene 2.9. This API
-	/// has moved from being <see cref="Token" /> based to <see cref="IAttribute" /> based. While
-	/// <see cref="Token" /> still exists in 2.9 as a convenience class, the preferred way
-	/// to store the information of a <see cref="Token" /> is to use <see cref="Util.Attribute" />s.
-	/// <p/>
-	/// <c>TokenStream</c> now extends <see cref="AttributeSource" />, which provides
-	/// access to all of the token <see cref="IAttribute" />s for the <c>TokenStream</c>.
-	/// Note that only one instance per <see cref="Util.Attribute" /> is created and reused
-	/// for every token. This approach reduces object creation and allows local
-	/// caching of references to the <see cref="Util.Attribute" />s. See
-	/// <see cref="IncrementToken()" /> for further details.
-	/// <p/>
-	/// <b>The workflow of the new <c>TokenStream</c> API is as follows:</b>
-	/// <list type="bullet">
-	/// <item>Instantiation of <c>TokenStream</c>/<see cref="TokenFilter" />s which add/get
-	/// attributes to/from the <see cref="AttributeSource" />.</item>
-	/// <item>The consumer calls <see cref="TokenStream.Reset()" />.</item>
-	/// <item>The consumer retrieves attributes from the stream and stores local
-	/// references to all attributes it wants to access</item>
-	/// <item>The consumer calls <see cref="IncrementToken()" /> until it returns false and
-	/// consumes the attributes after each call.</item>
-	/// <item>The consumer calls <see cref="End()" /> so that any end-of-stream operations
-	/// can be performed.</item>
-	/// <item>The consumer calls <see cref="Close()" /> to release any resource when finished
-	/// using the <c>TokenStream</c></item>
-	/// </list>
-	/// To make sure that filters and consumers know which attributes are available,
-	/// the attributes must be added during instantiation. Filters and consumers are
-	/// not required to check for availability of attributes in
-	/// <see cref="IncrementToken()" />.
-	/// <p/>
-	/// You can find some example code for the new API in the analysis package level
-	/// Javadoc.
-	/// <p/>
-	/// Sometimes it is desirable to capture a current state of a <c>TokenStream</c>
-	/// , e. g. for buffering purposes (see <see cref="CachingTokenFilter" />,
-	/// <see cref="TeeSinkTokenFilter" />). For this usecase
-	/// <see cref="AttributeSource.CaptureState" /> and <see cref="AttributeSource.RestoreState" />
-	/// can be used.
-	/// </summary>
-	public abstract class TokenStream : AttributeSource, IDisposable
-	{
-		/// <summary> A TokenStream using the default attribute factory.</summary>
-		protected internal TokenStream()
-		{ }
-		
-		/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
+    
+    /// <summary> A <c>TokenStream</c> enumerates the sequence of tokens, either from
+    /// <see cref="Field" />s of a <see cref="Document" /> or from query text.
+    /// <p/>
+    /// This is an abstract class. Concrete subclasses are:
+    /// <list type="bullet">
+    /// <item><see cref="Tokenizer" />, a <c>TokenStream</c> whose input is a Reader; and</item>
+    /// <item><see cref="TokenFilter" />, a <c>TokenStream</c> whose input is another
+    /// <c>TokenStream</c>.</item>
+    /// </list>
+    /// A new <c>TokenStream</c> API has been introduced with Lucene 2.9. This API
+    /// has moved from being <see cref="Token" /> based to <see cref="IAttribute" /> based. While
+    /// <see cref="Token" /> still exists in 2.9 as a convenience class, the preferred way
+    /// to store the information of a <see cref="Token" /> is to use <see cref="Util.Attribute" />s.
+    /// <p/>
+    /// <c>TokenStream</c> now extends <see cref="AttributeSource" />, which provides
+    /// access to all of the token <see cref="IAttribute" />s for the <c>TokenStream</c>.
+    /// Note that only one instance per <see cref="Util.Attribute" /> is created and reused
+    /// for every token. This approach reduces object creation and allows local
+    /// caching of references to the <see cref="Util.Attribute" />s. See
+    /// <see cref="IncrementToken()" /> for further details.
+    /// <p/>
+    /// <b>The workflow of the new <c>TokenStream</c> API is as follows:</b>
+    /// <list type="bullet">
+    /// <item>Instantiation of <c>TokenStream</c>/<see cref="TokenFilter" />s which add/get
+    /// attributes to/from the <see cref="AttributeSource" />.</item>
+    /// <item>The consumer calls <see cref="TokenStream.Reset()" />.</item>
+    /// <item>The consumer retrieves attributes from the stream and stores local
+    /// references to all attributes it wants to access</item>
+    /// <item>The consumer calls <see cref="IncrementToken()" /> until it returns false and
+    /// consumes the attributes after each call.</item>
+    /// <item>The consumer calls <see cref="End()" /> so that any end-of-stream operations
+    /// can be performed.</item>
+    /// <item>The consumer calls <see cref="Close()" /> to release any resource when finished
+    /// using the <c>TokenStream</c></item>
+    /// </list>
+    /// To make sure that filters and consumers know which attributes are available,
+    /// the attributes must be added during instantiation. Filters and consumers are
+    /// not required to check for availability of attributes in
+    /// <see cref="IncrementToken()" />.
+    /// <p/>
+    /// You can find some example code for the new API in the analysis package level
+    /// Javadoc.
+    /// <p/>
+    /// Sometimes it is desirable to capture a current state of a <c>TokenStream</c>
+    /// , e. g. for buffering purposes (see <see cref="CachingTokenFilter" />,
+    /// <see cref="TeeSinkTokenFilter" />). For this usecase
+    /// <see cref="AttributeSource.CaptureState" /> and <see cref="AttributeSource.RestoreState" />
+    /// can be used.
+    /// </summary>
+    public abstract class TokenStream : AttributeSource, IDisposable
+    {
+        /// <summary> A TokenStream using the default attribute factory.</summary>
+        protected internal TokenStream()
+        { }
+        
+        /// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
         protected internal TokenStream(AttributeSource input)
             : base(input)
-		{ }
-		
-		/// <summary> A TokenStream using the supplied AttributeFactory for creating new <see cref="IAttribute" /> instances.</summary>
+        { }
+        
+        /// <summary> A TokenStream using the supplied AttributeFactory for creating new <see cref="IAttribute" /> instances.</summary>
         protected internal TokenStream(AttributeFactory factory)
             : base(factory)
-		{ }
+        { }
 
-	    /// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
-	    /// the next token. Implementing classes must implement this method and update
-	    /// the appropriate <see cref="Util.Attribute" />s with the attributes of the next
-	    /// token.
-	    /// 
-	    /// The producer must make no assumptions about the attributes after the
-	    /// method has been returned: the caller may arbitrarily change it. If the
-	    /// producer needs to preserve the state for subsequent calls, it can use
-	    /// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
-	    /// 
-	    /// This method is called for every token of a document, so an efficient
-	    /// implementation is crucial for good performance. To avoid calls to
-	    /// <see cref="AttributeSource.AddAttribute{T}()" /> and <see cref="AttributeSource.GetAttribute{T}()" />,
-	    /// references to all <see cref="Util.Attribute" />s that this stream uses should be
-	    /// retrieved during instantiation.
-	    /// 
-	    /// To ensure that filters and consumers know which attributes are available,
-	    /// the attributes must be added during instantiation. Filters and consumers
-	    /// are not required to check for availability of attributes in
-	    /// <see cref="IncrementToken()" />.
-	    /// 
-	    /// </summary>
-	    /// <returns> false for end of stream; true otherwise</returns>
-	    public abstract bool IncrementToken();
-		
-		/// <summary> This method is called by the consumer after the last token has been
-		/// consumed, after <see cref="IncrementToken" /> returned <c>false</c>
-		/// (using the new <c>TokenStream</c> API). Streams implementing the old API
-		/// should upgrade to use this feature.
-		/// <p/>
-		/// This method can be used to perform any end-of-stream operations, such as
-		/// setting the final offset of a stream. The final offset of a stream might
-		/// differ from the offset of the last token eg in case one or more whitespaces
-		/// followed after the last token, but a <see cref="WhitespaceTokenizer" /> was used.
-		/// 
-		/// </summary>
-		/// <throws>  IOException </throws>
-		public virtual void  End()
-		{
-			// do nothing by default
-		}
-		
-		/// <summary> Resets this stream to the beginning. This is an optional operation, so
-		/// subclasses may or may not implement this method. <see cref="Reset()" /> is not needed for
-		/// the standard indexing process. However, if the tokens of a
-		/// <c>TokenStream</c> are intended to be consumed more than once, it is
-		/// necessary to implement <see cref="Reset()" />. Note that if your TokenStream
-		/// caches tokens and feeds them back again after a reset, it is imperative
-		/// that you clone the tokens when you store them away (on the first pass) as
-		/// well as when you return them (on future passes after <see cref="Reset()" />).
-		/// </summary>
-		public virtual void  Reset()
-		{
-		}
-		
-		/// <summary>Releases resources associated with this stream. </summary>
-		[Obsolete("Use Dispose() instead")]
-		public void  Close()
-		{
+        /// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
+        /// the next token. Implementing classes must implement this method and update
+        /// the appropriate <see cref="Util.Attribute" />s with the attributes of the next
+        /// token.
+        /// 
+        /// The producer must make no assumptions about the attributes after the
+        /// method has been returned: the caller may arbitrarily change it. If the
+        /// producer needs to preserve the state for subsequent calls, it can use
+        /// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
+        /// 
+        /// This method is called for every token of a document, so an efficient
+        /// implementation is crucial for good performance. To avoid calls to
+        /// <see cref="AttributeSource.AddAttribute{T}()" /> and <see cref="AttributeSource.GetAttribute{T}()" />,
+        /// references to all <see cref="Util.Attribute" />s that this stream uses should be
+        /// retrieved during instantiation.
+        /// 
+        /// To ensure that filters and consumers know which attributes are available,
+        /// the attributes must be added during instantiation. Filters and consumers
+        /// are not required to check for availability of attributes in
+        /// <see cref="IncrementToken()" />.
+        /// 
+        /// </summary>
+        /// <returns> false for end of stream; true otherwise</returns>
+        public abstract bool IncrementToken();
+        
+        /// <summary> This method is called by the consumer after the last token has been
+        /// consumed, after <see cref="IncrementToken" /> returned <c>false</c>
+        /// (using the new <c>TokenStream</c> API). Streams implementing the old API
+        /// should upgrade to use this feature.
+        /// <p/>
+        /// This method can be used to perform any end-of-stream operations, such as
+        /// setting the final offset of a stream. The final offset of a stream might
+        /// differ from the offset of the last token eg in case one or more whitespaces
+        /// followed after the last token, but a <see cref="WhitespaceTokenizer" /> was used.
+        /// 
+        /// </summary>
+        /// <throws>  IOException </throws>
+        public virtual void  End()
+        {
+            // do nothing by default
+        }
+        
+        /// <summary> Resets this stream to the beginning. This is an optional operation, so
+        /// subclasses may or may not implement this method. <see cref="Reset()" /> is not needed for
+        /// the standard indexing process. However, if the tokens of a
+        /// <c>TokenStream</c> are intended to be consumed more than once, it is
+        /// necessary to implement <see cref="Reset()" />. Note that if your TokenStream
+        /// caches tokens and feeds them back again after a reset, it is imperative
+        /// that you clone the tokens when you store them away (on the first pass) as
+        /// well as when you return them (on future passes after <see cref="Reset()" />).
+        /// </summary>
+        public virtual void  Reset()
+        {
+        }
+        
+        /// <summary>Releases resources associated with this stream. </summary>
+        [Obsolete("Use Dispose() instead")]
+        public void  Close()
+        {
             Dispose();
-		}
+        }
 
         public void Dispose()
         {
             Dispose(true);
         }
 
-	    protected abstract void Dispose(bool disposing);
-	}
+        protected abstract void Dispose(bool disposing);
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/FlagsAttribute.cs b/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
index b5c4b7b..afff36d 100644
--- a/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
@@ -20,66 +20,66 @@ using Attribute = Lucene.Net.Util.Attribute;
 
 namespace Lucene.Net.Analysis.Tokenattributes
 {
-	
-	/// <summary> This attribute can be used to pass different flags down the tokenizer chain,
-	/// eg from one TokenFilter to another one. 
-	/// </summary>
-	[Serializable]
-	public class FlagsAttribute:Util.Attribute, IFlagsAttribute, System.ICloneable
-	{
-		private int flags = 0;
+    
+    /// <summary> This attribute can be used to pass different flags down the tokenizer chain,
+    /// eg from one TokenFilter to another one. 
+    /// </summary>
+    [Serializable]
+    public class FlagsAttribute:Util.Attribute, IFlagsAttribute, System.ICloneable
+    {
+        private int flags = 0;
 
-	    /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
-	    /// <p/>
-	    /// 
-	    /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
-	    /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
-	    /// 
-	    /// 
-	    /// </summary>
-	    /// <value> The bits </value>
-	    public virtual int Flags
-	    {
-	        get { return flags; }
-	        set { this.flags = value; }
-	    }
+        /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
+        /// <p/>
+        /// 
+        /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
+        /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
+        /// 
+        /// 
+        /// </summary>
+        /// <value> The bits </value>
+        public virtual int Flags
+        {
+            get { return flags; }
+            set { this.flags = value; }
+        }
 
-	    public override void  Clear()
-		{
-			flags = 0;
-		}
-		
-		public  override bool Equals(System.Object other)
-		{
-			if (this == other)
-			{
-				return true;
-			}
-			
-			if (other is FlagsAttribute)
-			{
-				return ((FlagsAttribute) other).flags == flags;
-			}
-			
-			return false;
-		}
-		
-		public override int GetHashCode()
-		{
-			return flags;
-		}
-		
-		public override void  CopyTo(Attribute target)
-		{
-			IFlagsAttribute t = (IFlagsAttribute) target;
-			t.Flags = flags;
-		}
-		
-		override public System.Object Clone()
-		{
+        public override void  Clear()
+        {
+            flags = 0;
+        }
+        
+        public  override bool Equals(System.Object other)
+        {
+            if (this == other)
+            {
+                return true;
+            }
+            
+            if (other is FlagsAttribute)
+            {
+                return ((FlagsAttribute) other).flags == flags;
+            }
+            
+            return false;
+        }
+        
+        public override int GetHashCode()
+        {
+            return flags;
+        }
+        
+        public override void  CopyTo(Attribute target)
+        {
+            IFlagsAttribute t = (IFlagsAttribute) target;
+            t.Flags = flags;
+        }
+        
+        override public System.Object Clone()
+        {
             FlagsAttribute impl = new FlagsAttribute();
             impl.flags = this.flags;
             return impl;
-		}
-	}
+        }
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs b/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
index 24b2bea..325038e 100644
--- a/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
@@ -21,21 +21,21 @@ using Tokenizer = Lucene.Net.Analysis.Tokenizer;
 
 namespace Lucene.Net.Analysis.Tokenattributes
 {
-	
-	/// <summary> This attribute can be used to pass different flags down the <see cref="Tokenizer" /> chain,
-	/// eg from one TokenFilter to another one. 
-	/// </summary>
-	public interface IFlagsAttribute:IAttribute
-	{
-	    /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
-	    /// <p/>
-	    /// 
-	    /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
-	    /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
-	    /// 
-	    /// 
-	    /// </summary>
-	    /// <value> The bits </value>
-	    int Flags { get; set; }
-	}
+    
+    /// <summary> This attribute can be used to pass different flags down the <see cref="Tokenizer" /> chain,
+    /// eg from one TokenFilter to another one. 
+    /// </summary>
+    public interface IFlagsAttribute:IAttribute
+    {
+        /// <summary> EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
+        /// <p/>
+        /// 
+        /// Get the bitset for any bits that have been set.  This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
+        /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
+        /// 
+        /// 
+        /// </summary>
+        /// <value> The bits </value>
+        int Flags { get; set; }
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs b/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
index ffbbe02..3b00369 100644
--- a/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
@@ -20,29 +20,29 @@ using Lucene.Net.Util;
 
 namespace Lucene.Net.Analysis.Tokenattributes
 {
-	
-	/// <summary> The start and end character offset of a Token. </summary>
-	public interface IOffsetAttribute : IAttribute
-	{
-	    /// <summary>Returns this Token's starting offset, the position of the first character
-	    /// corresponding to this token in the source text.
-	    /// Note that the difference between endOffset() and startOffset() may not be
-	    /// equal to termText.length(), as the term text may have been altered by a
-	    /// stemmer or some other filter. 
-	    /// </summary>
-	    int StartOffset { get; }
+    
+    /// <summary> The start and end character offset of a Token. </summary>
+    public interface IOffsetAttribute : IAttribute
+    {
+        /// <summary>Returns this Token's starting offset, the position of the first character
+        /// corresponding to this token in the source text.
+        /// Note that the difference between endOffset() and startOffset() may not be
+        /// equal to termText.length(), as the term text may have been altered by a
+        /// stemmer or some other filter. 
+        /// </summary>
+        int StartOffset { get; }
 
 
-	    /// <summary>Set the starting and ending offset.
+        /// <summary>Set the starting and ending offset.
         /// See StartOffset() and EndOffset()
         /// </summary>
-		void  SetOffset(int startOffset, int endOffset);
+        void  SetOffset(int startOffset, int endOffset);
 
 
-	    /// <summary>Returns this Token's ending offset, one greater than the position of the
-	    /// last character corresponding to this token in the source text. The length
-	    /// of the token in the source text is (endOffset - startOffset). 
-	    /// </summary>
-	    int EndOffset { get; }
-	}
+        /// <summary>Returns this Token's ending offset, one greater than the position of the
+        /// last character corresponding to this token in the source text. The length
+        /// of the token in the source text is (endOffset - startOffset). 
+        /// </summary>
+        int EndOffset { get; }
+    }
 }
\ No newline at end of file


Mime
View raw message