lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ccurr...@apache.org
Subject [Lucene.Net] svn commit: r1294875 [11/45] - in /incubator/lucene.net/trunk: ./ build/ build/vs2010/contrib/ build/vs2010/test/ doc/ src/ src/contrib/Analyzers/ src/contrib/Analyzers/AR/ src/contrib/Analyzers/BR/ src/contrib/Analyzers/CJK/ src/contrib/Analyzers/Cn/ ...
Date Tue, 28 Feb 2012 22:43:28 GMT
Modified: incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.cs Tue Feb 28 22:43:08 2012
@@ -18,12 +18,13 @@
 
 
 /*
-
-NOTE: if you change this file and need to regenerate the tokenizer,
-remember to use JRE 1.4 when running jflex (before Lucene 3.0).
-This grammar now uses constructs (eg :digit:) whose meaning can
-vary according to the JRE used to run jflex.  See
-https://issues.apache.org/jira/browse/LUCENE-1126 for details
+    NOTE: if you change StandardTokenizerImpl.jflex and need to regenerate the tokenizer,
+    the tokenizer, only use Java 1.4 !!!
+    This grammar currently uses constructs (eg :digit:, :letter:) whose
+    meaning can vary according to the JRE used to run jflex.  See
+    https://issues.apache.org/jira/browse/LUCENE-1126 for details.
+    For current backwards compatibility it is needed to support
+    only Java 1.4 - this will change in Lucene 3.1.
 */
 
 using System;
@@ -246,10 +247,9 @@ namespace Lucene.Net.Analysis.Standard
 		public static readonly int NUM;
 		public static readonly int CJ;
 		/// <deprecated> this solves a bug where HOSTs that end with '.' are identified
-		/// as ACRONYMs. It is deprecated and will be removed in the next
-		/// release.
+		/// as ACRONYMs.
 		/// </deprecated>
-        [Obsolete("this solves a bug where HOSTs that end with '.' are identified as ACRONYMs. It is deprecated and will be removed in the next release.")]
+        [Obsolete("this solves a bug where HOSTs that end with '.' are identified as ACRONYMs")]
 		public static readonly int ACRONYM_DEP;
 		
 		public static readonly System.String[] TOKEN_TYPES;

Modified: incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.jflex?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.jflex (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/Standard/StandardTokenizerImpl.jflex Tue Feb 28 22:43:08 2012
@@ -1,156 +1,156 @@
-package org.apache.lucene.analysis.standard;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
-
-NOTE: if you change StandardTokenizerImpl.jflex and need to regenerate
-      the tokenizer, remember to use JRE 1.4 to run jflex (before
-      Lucene 3.0).  This grammar now uses constructs (eg :digit:,
-      :letter:) whose meaning can vary according to the JRE used to
-      run jflex.  See
-      https://issues.apache.org/jira/browse/LUCENE-1126 for details.
-
-*/
-
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
-%%
-
-%class StandardTokenizerImpl
-%unicode
-%integer
-%function getNextToken
-%pack
-%char
-
-%{
-
-public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
-public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
-public static final int ACRONYM           = StandardTokenizer.ACRONYM;
-public static final int COMPANY           = StandardTokenizer.COMPANY;
-public static final int EMAIL             = StandardTokenizer.EMAIL;
-public static final int HOST              = StandardTokenizer.HOST;
-public static final int NUM               = StandardTokenizer.NUM;
-public static final int CJ                = StandardTokenizer.CJ;
-/**
- * @deprecated this solves a bug where HOSTs that end with '.' are identified
- *             as ACRONYMs. It is deprecated and will be removed in the next
- *             release.
- */
-public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
-
-public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
-
-public final int yychar()
-{
-    return yychar;
-}
-
-/**
- * Resets the Tokenizer to a new Reader.
- */
-final void reset(java.io.Reader r) {
-  // reset to default buffer size, if buffer has grown
-  if (zzBuffer.length > ZZ_BUFFERSIZE) {
-    zzBuffer = new char[ZZ_BUFFERSIZE];
-  }
-  yyreset(r);
-}
-
-/**
- * Fills Lucene token with the current token text.
- */
-final void getText(Token t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
-}
-
-/**
- * Fills TermAttribute with the current token text.
- */
-final void getText(TermAttribute t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
-}
-
-%}
-
-THAI       = [\u0E00-\u0E59]
-
-// basic word: a sequence of digits & letters (includes Thai to enable ThaiAnalyzer to function)
-ALPHANUM   = ({LETTER}|{THAI}|[:digit:])+
-
-// internal apostrophes: O'Reilly, you're, O'Reilly's
-// use a post-filter to remove possessives
-APOSTROPHE =  {ALPHA} ("'" {ALPHA})+
-
-// acronyms: U.S.A., I.B.M., etc.
-// use a post-filter to remove dots
-ACRONYM    =  {LETTER} "." ({LETTER} ".")+
-
-ACRONYM_DEP	= {ALPHANUM} "." ({ALPHANUM} ".")+
-
-// company names like AT&T and Excite@Home.
-COMPANY    =  {ALPHA} ("&"|"@") {ALPHA}
-
-// email addresses
-EMAIL      =  {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
-
-// hostname
-HOST       =  {ALPHANUM} ((".") {ALPHANUM})+
-
-// floating point, serial, model numbers, ip addresses, etc.
-// every other segment must have at least one digit
-NUM        = ({ALPHANUM} {P} {HAS_DIGIT}
-           | {HAS_DIGIT} {P} {ALPHANUM}
-           | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
-           | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
-           | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
-           | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+)
-
-// punctuation
-P	         = ("_"|"-"|"/"|"."|",")
-
-// at least one digit
-HAS_DIGIT  = ({LETTER}|[:digit:])* [:digit:] ({LETTER}|[:digit:])*
-
-ALPHA      = ({LETTER})+
-
-// From the JFlex manual: "the expression that matches everything of <a> not matched by <b> is !(!<a>|<b>)"
-LETTER     = !(![:letter:]|{CJ})
-
-// Chinese and Japanese (but NOT Korean, which is included in [:letter:])
-CJ         = [\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
-
-WHITESPACE = \r\n | [ \r\n\t\f]
-
-%%
-
-{ALPHANUM}                                                     { return ALPHANUM; }
-{APOSTROPHE}                                                   { return APOSTROPHE; }
-{ACRONYM}                                                      { return ACRONYM; }
-{COMPANY}                                                      { return COMPANY; }
-{EMAIL}                                                        { return EMAIL; }
-{HOST}                                                         { return HOST; }
-{NUM}                                                          { return NUM; }
-{CJ}                                                           { return CJ; }
-{ACRONYM_DEP}                                                  { return ACRONYM_DEP; }
-
-/** Ignore the rest */
-. | {WHITESPACE}                                               { /* ignore */ }
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+
+WARNING: if you change StandardTokenizerImpl.jflex and need to regenerate
+      the tokenizer, only use Java 1.4 !!!
+      This grammar currently uses constructs (eg :digit:, :letter:) whose 
+      meaning can vary according to the JRE used to run jflex.  See
+      https://issues.apache.org/jira/browse/LUCENE-1126 for details.
+      For current backwards compatibility it is needed to support
+      only Java 1.4 - this will change in Lucene 3.1.
+
+*/
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+%%
+
+%class StandardTokenizerImpl
+%unicode
+%integer
+%function getNextToken
+%pack
+%char
+
+%{
+
+public static final int ALPHANUM          = StandardTokenizer.ALPHANUM;
+public static final int APOSTROPHE        = StandardTokenizer.APOSTROPHE;
+public static final int ACRONYM           = StandardTokenizer.ACRONYM;
+public static final int COMPANY           = StandardTokenizer.COMPANY;
+public static final int EMAIL             = StandardTokenizer.EMAIL;
+public static final int HOST              = StandardTokenizer.HOST;
+public static final int NUM               = StandardTokenizer.NUM;
+public static final int CJ                = StandardTokenizer.CJ;
+/**
+ * @deprecated this solves a bug where HOSTs that end with '.' are identified
+ *             as ACRONYMs.
+ */
+public static final int ACRONYM_DEP       = StandardTokenizer.ACRONYM_DEP;
+
+public static final String [] TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
+
+public final int yychar()
+{
+    return yychar;
+}
+
+/**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+  // reset to default buffer size, if buffer has grown
+  if (zzBuffer.length > ZZ_BUFFERSIZE) {
+    zzBuffer = new char[ZZ_BUFFERSIZE];
+  }
+  yyreset(r);
+}
+
+/**
+ * Fills Lucene token with the current token text.
+ */
+final void getText(Token t) {
+  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+
+/**
+ * Fills TermAttribute with the current token text.
+ */
+final void getText(TermAttribute t) {
+  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+
+%}
+
+THAI       = [\u0E00-\u0E59]
+
+// basic word: a sequence of digits & letters (includes Thai to enable ThaiAnalyzer to function)
+ALPHANUM   = ({LETTER}|{THAI}|[:digit:])+
+
+// internal apostrophes: O'Reilly, you're, O'Reilly's
+// use a post-filter to remove possessives
+APOSTROPHE =  {ALPHA} ("'" {ALPHA})+
+
+// acronyms: U.S.A., I.B.M., etc.
+// use a post-filter to remove dots
+ACRONYM    =  {LETTER} "." ({LETTER} ".")+
+
+ACRONYM_DEP	= {ALPHANUM} "." ({ALPHANUM} ".")+
+
+// company names like AT&T and Excite@Home.
+COMPANY    =  {ALPHA} ("&"|"@") {ALPHA}
+
+// email addresses
+EMAIL      =  {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
+
+// hostname
+HOST       =  {ALPHANUM} ((".") {ALPHANUM})+
+
+// floating point, serial, model numbers, ip addresses, etc.
+// every other segment must have at least one digit
+NUM        = ({ALPHANUM} {P} {HAS_DIGIT}
+           | {HAS_DIGIT} {P} {ALPHANUM}
+           | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
+           | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+           | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+           | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+)
+
+// punctuation
+P	         = ("_"|"-"|"/"|"."|",")
+
+// at least one digit
+HAS_DIGIT  = ({LETTER}|[:digit:])* [:digit:] ({LETTER}|[:digit:])*
+
+ALPHA      = ({LETTER})+
+
+// From the JFlex manual: "the expression that matches everything of <a> not matched by <b> is !(!<a>|<b>)"
+LETTER     = !(![:letter:]|{CJ})
+
+// Chinese and Japanese (but NOT Korean, which is included in [:letter:])
+CJ         = [\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
+
+WHITESPACE = \r\n | [ \r\n\t\f]
+
+%%
+
+{ALPHANUM}                                                     { return ALPHANUM; }
+{APOSTROPHE}                                                   { return APOSTROPHE; }
+{ACRONYM}                                                      { return ACRONYM; }
+{COMPANY}                                                      { return COMPANY; }
+{EMAIL}                                                        { return EMAIL; }
+{HOST}                                                         { return HOST; }
+{NUM}                                                          { return NUM; }
+{CJ}                                                           { return CJ; }
+{ACRONYM_DEP}                                                  { return ACRONYM_DEP; }
+
+/** Ignore the rest */
+. | {WHITESPACE}                                               { /* ignore */ }

Modified: incubator/lucene.net/trunk/src/core/Analysis/StopAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/StopAnalyzer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/StopAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/StopAnalyzer.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,7 @@
  */
 
 using System;
-
+using System.Collections.Generic;
 using Version = Lucene.Net.Util.Version;
 
 namespace Lucene.Net.Analysis
@@ -36,157 +36,28 @@ namespace Lucene.Net.Analysis
 	
 	public sealed class StopAnalyzer:Analyzer
 	{
-		private System.Collections.Hashtable stopWords;
-		// @deprecated
-        [Obsolete]
-		private bool useDefaultStopPositionIncrement;
+		private ISet<string> stopWords;
 		private bool enablePositionIncrements;
-		
-		/// <summary>An array containing some common English words that are not usually useful
-		/// for searching. 
-		/// </summary>
-		/// <deprecated> Use <see cref="ENGLISH_STOP_WORDS_SET" /> instead 
-		/// </deprecated>
-        [Obsolete("Use ENGLISH_STOP_WORDS_SET instead ")]
-		public static readonly System.String[] ENGLISH_STOP_WORDS = new System.String[]{"a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"};
-		
+
 		/// <summary>An unmodifiable set containing some common English words that are not usually useful
 		/// for searching.
 		/// </summary>
-		public static System.Collections.Hashtable ENGLISH_STOP_WORDS_SET;
-		
-		/// <summary>Builds an analyzer which removes words in
-		/// ENGLISH_STOP_WORDS.
-		/// </summary>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version) instead")]
-		public StopAnalyzer()
-		{
-			stopWords = ENGLISH_STOP_WORDS_SET;
-			useDefaultStopPositionIncrement = true;
-			enablePositionIncrements = false;
-		}
+        public static ISet<string> ENGLISH_STOP_WORDS_SET;
 		
 		/// <summary> Builds an analyzer which removes words in ENGLISH_STOP_WORDS.</summary>
 		public StopAnalyzer(Version matchVersion)
 		{
 			stopWords = ENGLISH_STOP_WORDS_SET;
-			useDefaultStopPositionIncrement = false;
 			enablePositionIncrements = StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion);
 		}
-		
-		/// <summary>Builds an analyzer which removes words in
-		/// ENGLISH_STOP_WORDS.
-		/// </summary>
-		/// <param name="enablePositionIncrements">
-		/// See <see cref="StopFilter.SetEnablePositionIncrements" />
-		/// </param>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version) instead")]
-		public StopAnalyzer(bool enablePositionIncrements)
-		{
-			stopWords = ENGLISH_STOP_WORDS_SET;
-			this.enablePositionIncrements = enablePositionIncrements;
-			useDefaultStopPositionIncrement = false;
-		}
-		
-		/// <summary>Builds an analyzer with the stop words from the given set.</summary>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.Collections.Hashtable)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, System.Collections.Hashtable) instead")]
-		public StopAnalyzer(System.Collections.Hashtable stopWords)
-		{
-			this.stopWords = stopWords;
-			useDefaultStopPositionIncrement = true;
-			enablePositionIncrements = false;
-		}
-		
+
 		/// <summary>Builds an analyzer with the stop words from the given set.</summary>
-		public StopAnalyzer(Version matchVersion, System.Collections.Hashtable stopWords)
+		public StopAnalyzer(Version matchVersion, ISet<string> stopWords)
 		{
 			this.stopWords = stopWords;
-			useDefaultStopPositionIncrement = false;
 			enablePositionIncrements = StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion);
 		}
 		
-		/// <summary>Builds an analyzer with the stop words from the given set.</summary>
-		/// <param name="stopWords">Set of stop words
-		/// </param>
-		/// <param name="enablePositionIncrements">
-		/// See <see cref="StopFilter.SetEnablePositionIncrements" />
-		/// </param>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.Collections.Hashtable)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, System.Collections.Hashtable) instead")]
-		public StopAnalyzer(System.Collections.Hashtable stopWords, bool enablePositionIncrements)
-		{
-			this.stopWords = stopWords;
-			this.enablePositionIncrements = enablePositionIncrements;
-			useDefaultStopPositionIncrement = false;
-		}
-		
-		/// <summary>Builds an analyzer which removes words in the provided array.</summary>
-        /// <deprecated> Use <see cref="StopAnalyzer(System.Collections.Hashtable, bool)" /> instead 
-		/// </deprecated>
-        /// <deprecated> Use <see cref="StopAnalyzer(Version, System.Collections.Hashtable)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(System.Collections.Hashtable, boolean) or StopAnalyzer(Version, System.Collections.Hashtable) instead ")]
-		public StopAnalyzer(System.String[] stopWords)
-		{
-			this.stopWords = StopFilter.MakeStopSet(stopWords);
-			useDefaultStopPositionIncrement = true;
-			enablePositionIncrements = false;
-		}
-		
-		/// <summary>Builds an analyzer which removes words in the provided array.</summary>
-		/// <param name="stopWords">Array of stop words
-		/// </param>
-		/// <param name="enablePositionIncrements">
-		/// See <see cref="StopFilter.SetEnablePositionIncrements" />
-		/// </param>
-        /// <deprecated> Use <see cref="StopAnalyzer(Version, System.Collections.Hashtable)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, Set) instead")]
-		public StopAnalyzer(System.String[] stopWords, bool enablePositionIncrements)
-		{
-			this.stopWords = StopFilter.MakeStopSet(stopWords);
-			this.enablePositionIncrements = enablePositionIncrements;
-			useDefaultStopPositionIncrement = false;
-		}
-		
-		/// <summary>Builds an analyzer with the stop words from the given file.</summary>
-		/// <seealso cref="WordlistLoader.GetWordSet(System.IO.FileInfo)">
-		/// </seealso>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.IO.FileInfo)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, File) instead")]
-		public StopAnalyzer(System.IO.FileInfo stopwordsFile)
-		{
-			stopWords = WordlistLoader.GetWordSet(stopwordsFile);
-			useDefaultStopPositionIncrement = true;
-			enablePositionIncrements = false;
-		}
-		
-		/// <summary>Builds an analyzer with the stop words from the given file.</summary>
-		/// <seealso cref="WordlistLoader.GetWordSet(System.IO.FileInfo)">
-		/// </seealso>
-		/// <param name="stopwordsFile">File to load stop words from
-		/// </param>
-		/// <param name="enablePositionIncrements">
-		/// See <see cref="StopFilter.SetEnablePositionIncrements" />
-		/// </param>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.IO.FileInfo)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, File) instead")]
-		public StopAnalyzer(System.IO.FileInfo stopwordsFile, bool enablePositionIncrements)
-		{
-			stopWords = WordlistLoader.GetWordSet(stopwordsFile);
-			this.enablePositionIncrements = enablePositionIncrements;
-			useDefaultStopPositionIncrement = false;
-		}
-		
 		/// <summary> Builds an analyzer with the stop words from the given file.
 		/// 
 		/// </summary>
@@ -200,38 +71,6 @@ namespace Lucene.Net.Analysis
 		{
 			stopWords = WordlistLoader.GetWordSet(stopwordsFile);
 			this.enablePositionIncrements = StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion);
-			useDefaultStopPositionIncrement = false;
-		}
-		
-		/// <summary>Builds an analyzer with the stop words from the given reader.</summary>
-		/// <seealso cref="WordlistLoader.GetWordSet(System.IO.TextReader)">
-		/// </seealso>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.IO.TextReader)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, Reader) instead")]
-		public StopAnalyzer(System.IO.TextReader stopwords)
-		{
-			stopWords = WordlistLoader.GetWordSet(stopwords);
-			useDefaultStopPositionIncrement = true;
-			enablePositionIncrements = false;
-		}
-		
-		/// <summary>Builds an analyzer with the stop words from the given reader.</summary>
-		/// <seealso cref="WordlistLoader.GetWordSet(System.IO.TextReader)">
-		/// </seealso>
-		/// <param name="stopwords">Reader to load stop words from
-		/// </param>
-		/// <param name="enablePositionIncrements">
-		/// See <see cref="StopFilter.SetEnablePositionIncrements" />
-		/// </param>
-		/// <deprecated> Use <see cref="StopAnalyzer(Version, System.IO.TextReader)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopAnalyzer(Version, Reader) instead")]
-		public StopAnalyzer(System.IO.TextReader stopwords, bool enablePositionIncrements)
-		{
-			stopWords = WordlistLoader.GetWordSet(stopwords);
-			this.enablePositionIncrements = enablePositionIncrements;
-			useDefaultStopPositionIncrement = false;
 		}
 
         /// <summary>Builds an analyzer with the stop words from the given reader. </summary>
@@ -245,20 +84,12 @@ namespace Lucene.Net.Analysis
         {
             stopWords = WordlistLoader.GetWordSet(stopwords);
             this.enablePositionIncrements = StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion);
-            useDefaultStopPositionIncrement = false;
         }
 
         /// <summary>Filters LowerCaseTokenizer with StopFilter. </summary>
 		public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
 		{
-			if (useDefaultStopPositionIncrement)
-			{
-				return new StopFilter(new LowerCaseTokenizer(reader), stopWords);
-			}
-			else
-			{
-				return new StopFilter(enablePositionIncrements, new LowerCaseTokenizer(reader), stopWords);
-			}
+			return new StopFilter(enablePositionIncrements, new LowerCaseTokenizer(reader), stopWords);
 		}
 		
 		/// <summary>Filters LowerCaseTokenizer with StopFilter. </summary>
@@ -292,14 +123,7 @@ namespace Lucene.Net.Analysis
 			{
 				streams = new SavedStreams(this);
 				streams.source = new LowerCaseTokenizer(reader);
-				if (useDefaultStopPositionIncrement)
-				{
-					streams.result = new StopFilter(streams.source, stopWords);
-				}
-				else
-				{
-					streams.result = new StopFilter(enablePositionIncrements, streams.source, stopWords);
-				}
+				streams.result = new StopFilter(enablePositionIncrements, streams.source, stopWords);
 				SetPreviousTokenStream(streams);
 			}
 			else
@@ -311,7 +135,7 @@ namespace Lucene.Net.Analysis
 			{
 				System.String[] stopWords = new System.String[]{"a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"};
 				CharArraySet stopSet = new CharArraySet(stopWords.Length, false);
-				stopSet.AddAll(new System.Collections.ArrayList(stopWords));
+				stopSet.AddAll(stopWords);
 				ENGLISH_STOP_WORDS_SET = CharArraySet.UnmodifiableSet(stopSet);
 			}
 		}

Modified: incubator/lucene.net/trunk/src/core/Analysis/StopFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/StopFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/StopFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/StopFilter.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,8 @@
  */
 
 using System;
-
+using System.Collections.Generic;
+using Lucene.Net.Util;
 using PositionIncrementAttribute = Lucene.Net.Analysis.Tokenattributes.PositionIncrementAttribute;
 using TermAttribute = Lucene.Net.Analysis.Tokenattributes.TermAttribute;
 using QueryParser = Lucene.Net.QueryParsers.QueryParser;
@@ -29,95 +30,12 @@ namespace Lucene.Net.Analysis
 	
 	public sealed class StopFilter:TokenFilter
 	{
-		
-		// deprecated
-        [Obsolete]
-		private static bool ENABLE_POSITION_INCREMENTS_DEFAULT = false;
-		
 		private CharArraySet stopWords;
-		private bool enablePositionIncrements = ENABLE_POSITION_INCREMENTS_DEFAULT;
+		private bool enablePositionIncrements = false;
 		
 		private TermAttribute termAtt;
 		private PositionIncrementAttribute posIncrAtt;
 		
-		/// <summary> Construct a token stream filtering the given input.</summary>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, String[])" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, String[]) instead")]
-		public StopFilter(TokenStream input, System.String[] stopWords):this(ENABLE_POSITION_INCREMENTS_DEFAULT, input, stopWords, false)
-		{
-		}
-		
-		/// <summary> Construct a token stream filtering the given input.</summary>
-		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words
-		/// </param>
-		/// <param name="input">input TokenStream
-		/// </param>
-		/// <param name="stopWords">array of stop words
-		/// </param>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, System.Collections.Hashtable)" /> instead.
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, Hashtable) instead.")]
-		public StopFilter(bool enablePositionIncrements, TokenStream input, System.String[] stopWords):this(enablePositionIncrements, input, stopWords, false)
-		{
-		}
-		
-		/// <summary> Constructs a filter which removes words from the input
-		/// TokenStream that are named in the array of words.
-		/// </summary>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, String[], bool)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, String[], bool) instead")]
-		public StopFilter(TokenStream in_Renamed, System.String[] stopWords, bool ignoreCase):this(ENABLE_POSITION_INCREMENTS_DEFAULT, in_Renamed, stopWords, ignoreCase)
-		{
-		}
-		
-		/// <summary> Constructs a filter which removes words from the input
-		/// TokenStream that are named in the array of words.
-		/// </summary>
-		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words
-		/// </param>
-		///  <param name="in_Renamed">input TokenStream
-		/// </param>
-		/// <param name="stopWords">array of stop words
-		/// </param>
-		/// <param name="ignoreCase">true if case is ignored
-		/// </param>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, System.Collections.Hashtable, bool)" /> instead.
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, Hashtable, bool) instead.")]
-		public StopFilter(bool enablePositionIncrements, TokenStream in_Renamed, System.String[] stopWords, bool ignoreCase):base(in_Renamed)
-		{
-			this.stopWords = (CharArraySet) MakeStopSet(stopWords, ignoreCase);
-			this.enablePositionIncrements = enablePositionIncrements;
-			Init();
-		}
-		
-		
-		/// <summary> Construct a token stream filtering the given input.
-		/// If <c>stopWords</c> is an instance of <see cref="CharArraySet" /> (true if
-		/// <c>makeStopSet()</c> was used to construct the set) it will be directly used
-		/// and <c>ignoreCase</c> will be ignored since <c>CharArraySet</c>
-		/// directly controls case sensitivity.
-		/// <p/>
-		/// If <c>stopWords</c> is not an instance of <see cref="CharArraySet" />,
-		/// a new CharArraySet will be constructed and <c>ignoreCase</c> will be
-		/// used to specify the case sensitivity of that set.
-		/// 
-		/// </summary>
-		/// <param name="input">
-		/// </param>
-		/// <param name="stopWords">The set of Stop Words.
-		/// </param>
-		/// <param name="ignoreCase">-Ignore case when stopping.
-		/// </param>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, System.Collections.Hashtable, bool)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, Set, bool) instead")]
-		public StopFilter(TokenStream input, System.Collections.Hashtable stopWords, bool ignoreCase):this(ENABLE_POSITION_INCREMENTS_DEFAULT, input, stopWords, ignoreCase)
-		{
-		}
-		
 		/// <summary> Construct a token stream filtering the given input.
 		/// If <c>stopWords</c> is an instance of <see cref="CharArraySet" /> (true if
 		/// <c>makeStopSet()</c> was used to construct the set) it will be directly used
@@ -127,65 +45,38 @@ namespace Lucene.Net.Analysis
 		/// If <c>stopWords</c> is not an instance of <see cref="CharArraySet" />,
 		/// a new CharArraySet will be constructed and <c>ignoreCase</c> will be
 		/// used to specify the case sensitivity of that set.
-		/// 
 		/// </summary>
-		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words
-		/// </param>
-		/// <param name="input">Input TokenStream
-		/// </param>
-		/// <param name="stopWords">The set of Stop Words.
-		/// </param>
-		/// <param name="ignoreCase">-Ignore case when stopping.
-		/// </param>
-		public StopFilter(bool enablePositionIncrements, TokenStream input, System.Collections.Hashtable stopWords, bool ignoreCase):base(input)
-		{
-			if (stopWords is CharArraySet)
-			{
-				this.stopWords = (CharArraySet) stopWords;
-			}
-			else
-			{
-				this.stopWords = new CharArraySet(stopWords.Count, ignoreCase);
-				this.stopWords.Add(stopWords);
-			}
-			this.enablePositionIncrements = enablePositionIncrements;
-			Init();
+		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words</param>
+		/// <param name="input">Input TokenStream</param>
+		/// <param name="stopWords">A Set of strings or strings or char[] or any other ToString()-able set representing the stopwords</param>
+        /// <param name="ignoreCase">if true, all words are lower cased first</param>
+        public StopFilter(bool enablePositionIncrements, TokenStream input, ISet<string> stopWords, bool ignoreCase)
+            : base(input)
+		{
+		    if (stopWords is CharArraySet)
+		    {
+		        this.stopWords = (CharArraySet) stopWords;
+		    }
+		    else
+		    {
+		        this.stopWords = new CharArraySet(stopWords.Count, ignoreCase);
+		        this.stopWords.AddAll(stopWords);
+		    }
+		    this.enablePositionIncrements = enablePositionIncrements;
+		    termAtt = AddAttribute<TermAttribute>();
+            posIncrAtt = AddAttribute<PositionIncrementAttribute>();
 		}
-		
-		/// <summary> Constructs a filter which removes words from the input
+
+	    /// <summary> Constructs a filter which removes words from the input
 		/// TokenStream that are named in the Set.
-		/// 
 		/// </summary>
-		/// <seealso cref="MakeStopSet(String[])">
-		/// </seealso>
-		/// <deprecated> Use <see cref="StopFilter(bool, TokenStream, System.Collections.Hashtable)" /> instead
-		/// </deprecated>
-        [Obsolete("Use StopFilter(bool, TokenStream, Hashtable) instead")]
-		public StopFilter(TokenStream in_Renamed, System.Collections.Hashtable stopWords):this(ENABLE_POSITION_INCREMENTS_DEFAULT, in_Renamed, stopWords, false)
-		{
-		}
-		
-		/// <summary> Constructs a filter which removes words from the input
-		/// TokenStream that are named in the Set.
-		/// 
-		/// </summary>
-		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words
-		/// </param>
-		///  <param name="in_Renamed">Input stream
-		/// </param>
-		/// <param name="stopWords">The set of Stop Words.
-		/// </param>
-		/// <seealso cref="MakeStopSet(String[])">
-		/// </seealso>
-		public StopFilter(bool enablePositionIncrements, TokenStream in_Renamed, System.Collections.Hashtable stopWords):this(enablePositionIncrements, in_Renamed, stopWords, false)
-		{
-		}
-		
-		public void  Init()
-		{
-			termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
-			posIncrAtt = (PositionIncrementAttribute) AddAttribute(typeof(PositionIncrementAttribute));
-		}
+		/// <param name="enablePositionIncrements">true if token positions should record the removed stop words</param>
+		///  <param name="in_Renamed">Input stream</param>
+		/// <param name="stopWords">A Set of strings or char[] or any other ToString()-able set representing the stopwords</param>
+		/// <seealso cref="MakeStopSet(String[])"/>
+		public StopFilter(bool enablePositionIncrements, TokenStream in_Renamed, ISet<string> stopWords)
+            : this(enablePositionIncrements, in_Renamed, stopWords, false)
+		{ }
 		
 		/// <summary> Builds a Set from an array of stop words,
 		/// appropriate for passing into the StopFilter constructor.
@@ -195,7 +86,7 @@ namespace Lucene.Net.Analysis
 		/// </summary>
 		/// <seealso cref="MakeStopSet(String[], bool)"> passing false to ignoreCase
 		/// </seealso>
-		public static System.Collections.Hashtable MakeStopSet(System.String[] stopWords)
+		public static ISet<string> MakeStopSet(params string[] stopWords)
 		{
 			return MakeStopSet(stopWords, false);
 		}
@@ -204,40 +95,34 @@ namespace Lucene.Net.Analysis
 		/// appropriate for passing into the StopFilter constructor.
 		/// This permits this stopWords construction to be cached once when
 		/// an Analyzer is constructed.
-		/// 
 		/// </summary>
-		/// <seealso cref="MakeStopSet(String[], bool)"> passing false to ignoreCase
-		/// </seealso>
-		public static System.Collections.Hashtable MakeStopSet(System.Collections.IList stopWords)
+		/// <param name="stopWords">A list of strings or char[] or any other ToString()-able list representing the stop words</param>
+		/// <seealso cref="MakeStopSet(String[], bool)">passing false to ignoreCase</seealso>
+		public static ISet<string> MakeStopSet(IList<object> stopWords)
 		{
 			return MakeStopSet(stopWords, false);
 		}
 		
-		/// <summary> </summary>
-		/// <param name="stopWords">An array of stopwords
-		/// </param>
-		/// <param name="ignoreCase">If true, all words are lower cased first.  
-		/// </param>
-		/// <returns> a Set containing the words
-		/// </returns>
-		public static System.Collections.Hashtable MakeStopSet(System.String[] stopWords, bool ignoreCase)
+		/// <summary></summary>
+		/// <param name="stopWords">An array of stopwords</param>
+		/// <param name="ignoreCase">If true, all words are lower cased first.</param>
+		/// <returns> a Set containing the words</returns>
+		public static ISet<string> MakeStopSet(string[] stopWords, bool ignoreCase)
 		{
 			CharArraySet stopSet = new CharArraySet(stopWords.Length, ignoreCase);
-			stopSet.AddAll(new System.Collections.ArrayList(stopWords));
+		    stopSet.AddAll(stopWords);
 			return stopSet;
 		}
 		
 		/// <summary> </summary>
-		/// <param name="stopWords">A List of Strings representing the stopwords
-		/// </param>
-		/// <param name="ignoreCase">if true, all words are lower cased first
-		/// </param>
-		/// <returns> A Set containing the words
-		/// </returns>
-		public static System.Collections.Hashtable MakeStopSet(System.Collections.IList stopWords, bool ignoreCase)
+        /// <param name="stopWords">A List of Strings or char[] or any other toString()-able list representing the stopwords </param>
+		/// <param name="ignoreCase">if true, all words are lower cased first</param>
+		/// <returns>A Set (<see cref="CharArraySet"/>)containing the words</returns>
+		public static ISet<string> MakeStopSet(IList<object> stopWords, bool ignoreCase)
 		{
 			CharArraySet stopSet = new CharArraySet(stopWords.Count, ignoreCase);
-			stopSet.AddAll(stopWords);
+            foreach(var word in stopWords)
+                stopSet.Add(word.ToString());
 			return stopSet;
 		}
 		
@@ -258,57 +143,17 @@ namespace Lucene.Net.Analysis
 				}
 				skippedPositions += posIncrAtt.GetPositionIncrement();
 			}
-			// reached EOS -- return null
+			// reached EOS -- return false
 			return false;
 		}
 		
-		/// <seealso cref="SetEnablePositionIncrementsDefault(bool)">
-		/// </seealso>
-		/// <deprecated> Please specify this when you create the StopFilter
-		/// </deprecated>
-        [Obsolete("Please specify this when you create the StopFilter")]
-		public static bool GetEnablePositionIncrementsDefault()
-		{
-			return ENABLE_POSITION_INCREMENTS_DEFAULT;
-		}
-		
 		/// <summary> Returns version-dependent default for enablePositionIncrements. Analyzers
 		/// that embed StopFilter use this method when creating the StopFilter. Prior
-		/// to 2.9, this returns <see cref="GetEnablePositionIncrementsDefault" />. On 2.9
-		/// or later, it returns true.
+		/// to 2.9, this returns false. On 2.9 or later, it returns true.
 		/// </summary>
 		public static bool GetEnablePositionIncrementsVersionDefault(Version matchVersion)
 		{
-			if (matchVersion.OnOrAfter(Version.LUCENE_29))
-			{
-				return true;
-			}
-			else
-			{
-				return ENABLE_POSITION_INCREMENTS_DEFAULT;
-			}
-		}
-		
-		/// <summary> Set the default position increments behavior of every StopFilter created
-		/// from now on.
-		/// <p/>
-		/// Note: behavior of a single StopFilter instance can be modified with
-		/// <see cref="SetEnablePositionIncrements(bool)" />. This static method allows
-		/// control over behavior of classes using StopFilters internally, for
-        /// example <see cref="Lucene.Net.Analysis.Standard.StandardAnalyzer"/>
-		/// if used with the no-arg ctor.
-		/// <p/>
-		/// Default : false.
-		/// 
-		/// </summary>
-		/// <seealso cref="SetEnablePositionIncrements(bool)">
-		/// </seealso>
-		/// <deprecated> Please specify this when you create the StopFilter
-		/// </deprecated>
-        [Obsolete("Please specify this when you create the StopFilter")]
-		public static void  SetEnablePositionIncrementsDefault(bool defaultValue)
-		{
-			ENABLE_POSITION_INCREMENTS_DEFAULT = defaultValue;
+            return matchVersion.OnOrAfter(Version.LUCENE_29);
 		}
 		
 		/// <seealso cref="SetEnablePositionIncrements(bool)">

Modified: incubator/lucene.net/trunk/src/core/Analysis/TeeSinkTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/TeeSinkTokenFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/TeeSinkTokenFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/TeeSinkTokenFilter.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,7 @@
  */
 
 using System;
-
+using System.Collections.Generic;
 using AttributeImpl = Lucene.Net.Util.AttributeImpl;
 using AttributeSource = Lucene.Net.Util.AttributeSource;
 
@@ -74,7 +74,7 @@ namespace Lucene.Net.Analysis
 				return true;
 			}
 		}
-		private System.Collections.IList sinks = new System.Collections.ArrayList();
+		private LinkedList<WeakReference> sinks = new LinkedList<WeakReference>();
 		
 		/// <summary> Instantiates a new TeeSinkTokenFilter.</summary>
 		public TeeSinkTokenFilter(TokenStream input):base(input)
@@ -95,7 +95,7 @@ namespace Lucene.Net.Analysis
 		public SinkTokenStream NewSinkTokenStream(SinkFilter filter)
 		{
 			SinkTokenStream sink = new SinkTokenStream(this.CloneAttributes(), filter);
-			this.sinks.Add(new System.WeakReference(sink));
+			this.sinks.AddLast(new System.WeakReference(sink));
 			return sink;
 		}
 		
@@ -106,16 +106,16 @@ namespace Lucene.Net.Analysis
 		public void  AddSinkTokenStream(SinkTokenStream sink)
 		{
 			// check that sink has correct factory
-			if (!this.GetAttributeFactory().Equals(sink.GetAttributeFactory()))
+			if (!this.Factory.Equals(sink.Factory))
 			{
 				throw new System.ArgumentException("The supplied sink is not compatible to this tee");
 			}
 			// add eventually missing attribute impls to the existing sink
-            foreach (AttributeImpl impl in this.CloneAttributes().GetAttributeImplsIterator())
+            foreach (AttributeImpl impl in this.CloneAttributes().AttributeImpls)
             {
                 sink.AddAttributeImpl(impl);
             }
-			this.sinks.Add(new System.WeakReference(sink));
+			this.sinks.AddLast(new WeakReference(sink));
 		}
 		
 		/// <summary> <c>TeeSinkTokenFilter</c> passes all tokens to the added sinks
@@ -125,8 +125,9 @@ namespace Lucene.Net.Analysis
 		/// </summary>
 		public void  ConsumeAllTokens()
 		{
-			while (IncrementToken())
-				;
+            while (IncrementToken())
+            {
+            }
 		}
 		
 		public override bool IncrementToken()
@@ -135,9 +136,9 @@ namespace Lucene.Net.Analysis
 			{
 				// capture state lazily - maybe no SinkFilter accepts this state
 				AttributeSource.State state = null;
-				for (System.Collections.IEnumerator it = sinks.GetEnumerator(); it.MoveNext(); )
+				foreach(WeakReference wr in sinks)
 				{
-					SinkTokenStream sink = (SinkTokenStream) ((System.WeakReference) it.Current).Target;
+				    SinkTokenStream sink = (SinkTokenStream)wr.Target;
 					if (sink != null)
 					{
 						if (sink.Accept(this))
@@ -160,9 +161,9 @@ namespace Lucene.Net.Analysis
 		{
 			base.End();
 			AttributeSource.State finalState = CaptureState();
-			for (System.Collections.IEnumerator it = sinks.GetEnumerator(); it.MoveNext(); )
+			foreach(WeakReference wr in sinks)
 			{
-				SinkTokenStream sink = (SinkTokenStream) ((System.WeakReference) it.Current).Target;
+                SinkTokenStream sink = (SinkTokenStream)wr.Target;
 				if (sink != null)
 				{
 					sink.SetFinalState(finalState);
@@ -181,20 +182,21 @@ namespace Lucene.Net.Analysis
 			/// <summary> Called by <see cref="SinkTokenStream.Reset()" />. This method does nothing by default
 			/// and can optionally be overridden.
 			/// </summary>
-			public void  Reset()
+			public virtual void Reset()
 			{
 				// nothing to do; can be overridden
 			}
 		}
 		
-		public sealed class SinkTokenStream:TokenStream
+		public sealed class SinkTokenStream : TokenStream
 		{
-			private System.Collections.IList cachedStates = new System.Collections.ArrayList();
+            private LinkedList<AttributeSource.State> cachedStates = new LinkedList<AttributeSource.State>();
 			private AttributeSource.State finalState;
-			private System.Collections.IEnumerator it = null;
+			private IEnumerator<AttributeSource.State> it = null;
 			private SinkFilter filter;
-			
-			internal SinkTokenStream(AttributeSource source, SinkFilter filter):base(source)
+
+			internal SinkTokenStream(AttributeSource source, SinkFilter filter)
+                : base(source)
 			{
 				this.filter = filter;
 			}
@@ -210,7 +212,7 @@ namespace Lucene.Net.Analysis
 				{
 					throw new System.SystemException("The tee must be consumed before sinks are consumed.");
 				}
-				cachedStates.Add(state);
+				cachedStates.AddLast(state);
 			}
 			
 			internal /*private*/ void  SetFinalState(AttributeSource.State finalState)
@@ -231,7 +233,7 @@ namespace Lucene.Net.Analysis
 					return false;
 				}
 				
-				AttributeSource.State state = (State) it.Current;
+				AttributeSource.State state = it.Current;
 				RestoreState(state);
 				return true;
 			}
@@ -248,6 +250,11 @@ namespace Lucene.Net.Analysis
 			{
 				it = cachedStates.GetEnumerator();
 			}
+
+		    protected override void Dispose(bool disposing)
+		    {
+		        // Do nothing.
+		    }
 		}
 		
 		private static readonly SinkFilter ACCEPT_ALL_FILTER;

Modified: incubator/lucene.net/trunk/src/core/Analysis/Token.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/Token.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/Token.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/Token.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,8 @@
  */
 
 using System;
-
+using Lucene.Net.Support;
+using Lucene.Net.Util;
 using FlagsAttribute = Lucene.Net.Analysis.Tokenattributes.FlagsAttribute;
 using OffsetAttribute = Lucene.Net.Analysis.Tokenattributes.OffsetAttribute;
 using PayloadAttribute = Lucene.Net.Analysis.Tokenattributes.PayloadAttribute;
@@ -57,21 +58,8 @@ namespace Lucene.Net.Analysis
 	/// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
 	/// be used as convenience class that implements all <see cref="Attribute" />s, which is especially useful
 	/// to easily switch from the old to the new TokenStream API.
-	/// </summary>
-	/// <summary><br/><br/>
-	/// <p/><b>NOTE:</b> As of 2.3, Token stores the term text
-	/// internally as a malleable char[] termBuffer instead of
-	/// String termText.  The indexing code and core tokenizers
-	/// have been changed to re-use a single Token instance, changing
-	/// its buffer and other fields in-place as the Token is
-	/// processed.  This provides substantially better indexing
-	/// performance as it saves the GC cost of new'ing a Token and
-	/// String for every term.  The APIs that accept String
-	/// termText are still available but a warning about the
-	/// associated performance cost has been added (below).  The
-	/// <see cref="TermText()" /> method has been deprecated.<p/>
-	/// </summary>
-	/// <summary><p/>Tokenizers and TokenFilters should try to re-use a Token instance when
+	/// <br/><br/>
+	/// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
 	/// possible for best performance, by implementing the
 	/// <see cref="TokenStream.IncrementToken()" /> API.
 	/// Failing that, to create a new Token you should first use
@@ -135,68 +123,17 @@ namespace Lucene.Net.Analysis
 	[Serializable]
 	public class Token:AttributeImpl, System.ICloneable, TermAttribute, TypeAttribute, PositionIncrementAttribute, FlagsAttribute, OffsetAttribute, PayloadAttribute
 	{
-		
 		public const System.String DEFAULT_TYPE = "word";
 		
 		private static int MIN_BUFFER_SIZE = 10;
-		
-		/// <deprecated> We will remove this when we remove the
-		/// deprecated APIs 
-		/// </deprecated>
-        [Obsolete("We will remove this when we remove the deprecated APIs")]
-		private System.String termText;
-		
-		/// <summary> Characters for the term text.</summary>
-		/// <deprecated> This will be made private. Instead, use:
-		/// <see cref="TermBuffer()" />, 
-		/// <see cref="SetTermBuffer(char[], int, int)" />,
-		/// <see cref="SetTermBuffer(String)" />, or
-		/// <see cref="SetTermBuffer(String, int, int)" />
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: TermBuffer(), SetTermBuffer(char[], int, int), SetTermBuffer(String) or SetTermBuffer(String, int, int)")]
-		internal char[] termBuffer;
-		
-		/// <summary> Length of term text in the buffer.</summary>
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="TermLength()" />, or <see cref="SetTermLength(int)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: TermLength(), or setTermLength(int)")]
-		internal int termLength;
-		
-		/// <summary> Start in source text.</summary>
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="StartOffset()" />, or <see cref="SetStartOffset(int)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: StartOffset(), or SetStartOffset(int).")]
-		internal int startOffset;
-		
-		/// <summary> End in source text.</summary>
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="EndOffset()" />, or <see cref="SetEndOffset(int)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: EndOffset(), or SetEndOffset(int).")]
-		internal int endOffset;
-		
-		/// <summary> The lexical type of the token.</summary>
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="Type()" />, or <see cref="SetType(String)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: Type(), or SetType(String).")]
-		internal System.String type = DEFAULT_TYPE;
-		
+
+        private char[] termBuffer;
+		private int termLength;
+		private int startOffset, endOffset;
+		private string type = DEFAULT_TYPE;
 		private int flags;
-		
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="GetPayload()" />, or <see cref="SetPayload(Payload)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: GetPayload(), or SetPayload(Payload).")]
-		internal Payload payload;
-		
-		/// <deprecated> This will be made private. Instead, use:
-        /// <see cref="GetPositionIncrement()" />, or <see cref="SetPositionIncrement(int)"/>.
-		/// </deprecated>
-        [Obsolete("This will be made private. Instead, use: GetPositionIncrement(), or SetPositionIncrement(int).")]
-		internal int positionIncrement = 1;
+		private Payload payload;
+		private int positionIncrement = 1;
 		
 		/// <summary>Constructs a Token will null text. </summary>
 		public Token()
@@ -206,10 +143,8 @@ namespace Lucene.Net.Analysis
 		/// <summary>Constructs a Token with null text and start &amp; end
 		/// offsets.
 		/// </summary>
-		/// <param name="start">start offset in the source text
-		/// </param>
-		/// <param name="end">end offset in the source text 
-		/// </param>
+		/// <param name="start">start offset in the source text</param>
+		/// <param name="end">end offset in the source text</param>
 		public Token(int start, int end)
 		{
 			startOffset = start;
@@ -219,12 +154,9 @@ namespace Lucene.Net.Analysis
 		/// <summary>Constructs a Token with null text and start &amp; end
 		/// offsets plus the Token type.
 		/// </summary>
-		/// <param name="start">start offset in the source text
-		/// </param>
-		/// <param name="end">end offset in the source text
-		/// </param>
-		/// <param name="typ">the lexical type of this Token 
-		/// </param>
+		/// <param name="start">start offset in the source text</param>
+		/// <param name="end">end offset in the source text</param>
+		/// <param name="typ">the lexical type of this Token</param>
 		public Token(int start, int end, System.String typ)
 		{
 			startOffset = start;
@@ -235,12 +167,9 @@ namespace Lucene.Net.Analysis
 		/// <summary> Constructs a Token with null text and start &amp; end
 		/// offsets plus flags. NOTE: flags is EXPERIMENTAL.
 		/// </summary>
-		/// <param name="start">start offset in the source text
-		/// </param>
-		/// <param name="end">end offset in the source text
-		/// </param>
-		/// <param name="flags">The bits to set for this token
-		/// </param>
+		/// <param name="start">start offset in the source text</param>
+		/// <param name="end">end offset in the source text</param>
+		/// <param name="flags">The bits to set for this token</param>
 		public Token(int start, int end, int flags)
 		{
 			startOffset = start;
@@ -254,15 +183,12 @@ namespace Lucene.Net.Analysis
 		/// instead use the char[] termBuffer methods to set the
 		/// term text.
 		/// </summary>
-		/// <param name="text">term text
-		/// </param>
-		/// <param name="start">start offset
-		/// </param>
-		/// <param name="end">end offset
-		/// </param>
+		/// <param name="text">term text</param>
+		/// <param name="start">start offset</param>
+		/// <param name="end">end offset</param>
 		public Token(System.String text, int start, int end)
 		{
-			termText = text;
+		    SetTermBuffer(text);
 			startOffset = start;
 			endOffset = end;
 		}
@@ -272,17 +198,13 @@ namespace Lucene.Net.Analysis
 		/// speed you should instead use the char[] termBuffer
 		/// methods to set the term text.
 		/// </summary>
-		/// <param name="text">term text
-		/// </param>
-		/// <param name="start">start offset
-		/// </param>
-		/// <param name="end">end offset
-		/// </param>
-		/// <param name="typ">token type
-		/// </param>
+		/// <param name="text">term text</param>
+		/// <param name="start">start offset</param>
+		/// <param name="end">end offset</param>
+		/// <param name="typ">token type</param>
 		public Token(System.String text, int start, int end, System.String typ)
 		{
-			termText = text;
+		    SetTermBuffer(text);
 			startOffset = start;
 			endOffset = end;
 			type = typ;
@@ -293,17 +215,13 @@ namespace Lucene.Net.Analysis
 		/// speed you should instead use the char[] termBuffer
 		/// methods to set the term text.
 		/// </summary>
-		/// <param name="text">
-		/// </param>
-		/// <param name="start">
-		/// </param>
-		/// <param name="end">
-		/// </param>
-		/// <param name="flags">token type bits
-		/// </param>
+		/// <param name="text"></param>
+		/// <param name="start"></param>
+		/// <param name="end"></param>
+		/// <param name="flags">token type bits</param>
 		public Token(System.String text, int start, int end, int flags)
 		{
-			termText = text;
+		    SetTermBuffer(text);
 			startOffset = start;
 			endOffset = end;
 			this.flags = flags;
@@ -313,16 +231,11 @@ namespace Lucene.Net.Analysis
 		/// &amp; length), start and end
 		/// offsets
 		/// </summary>
-		/// <param name="startTermBuffer">
-		/// </param>
-		/// <param name="termBufferOffset">
-		/// </param>
-		/// <param name="termBufferLength">
-		/// </param>
-		/// <param name="start">
-		/// </param>
-		/// <param name="end">
-		/// </param>
+		/// <param name="startTermBuffer"></param>
+		/// <param name="termBufferOffset"></param>
+		/// <param name="termBufferLength"></param>
+		/// <param name="start"></param>
+		/// <param name="end"></param>
 		public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
 		{
 			SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
@@ -373,37 +286,6 @@ namespace Lucene.Net.Analysis
 			return positionIncrement;
 		}
 		
-		/// <summary>Sets the Token's term text.  <b>NOTE:</b> for better
-		/// indexing speed you should instead use the char[]
-		/// termBuffer methods to set the term text.
-		/// </summary>
-		/// <deprecated> use <see cref="SetTermBuffer(char[], int, int)" /> or
-		/// <see cref="SetTermBuffer(String)" /> or
-		/// <see cref="SetTermBuffer(String, int, int)" />.
-		/// </deprecated>
-        [Obsolete("Use SetTermBuffer(char[], int, int) or SetTermBuffer(String) or SetTermBuffer(String, int, int)")]
-		public virtual void  SetTermText(System.String text)
-		{
-			termText = text;
-			termBuffer = null;
-		}
-		
-		/// <summary>Returns the Token's term text.
-		/// 
-		/// </summary>
-		/// <deprecated> This method now has a performance penalty
-		/// because the text is stored internally in a char[].  If
-		/// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
-		/// directly instead.  If you really need a
-		/// String, use <see cref="Term()" />
-		/// </deprecated>
-		public System.String TermText()
-		{
-			if (termText == null && termBuffer != null)
-				termText = new System.String(termBuffer, 0, termLength);
-			return termText;
-		}
-		
 		/// <summary>Returns the Token's term text.
 		/// 
 		/// This method has a performance penalty
@@ -415,8 +297,6 @@ namespace Lucene.Net.Analysis
 		/// </summary>
 		public System.String Term()
 		{
-			if (termText != null)
-				return termText;
 			InitTermBuffer();
 			return new System.String(termBuffer, 0, termLength);
 		}
@@ -424,15 +304,11 @@ namespace Lucene.Net.Analysis
 		/// <summary>Copies the contents of buffer, starting at offset for
 		/// length characters, into the termBuffer array.
 		/// </summary>
-		/// <param name="buffer">the buffer to copy
-		/// </param>
-		/// <param name="offset">the index in the buffer of the first character to copy
-		/// </param>
-		/// <param name="length">the number of characters to copy
-		/// </param>
+		/// <param name="buffer">the buffer to copy</param>
+		/// <param name="offset">the index in the buffer of the first character to copy</param>
+		/// <param name="length">the number of characters to copy</param>
 		public void  SetTermBuffer(char[] buffer, int offset, int length)
 		{
-			termText = null;
 			GrowTermBuffer(length);
 			Array.Copy(buffer, offset, termBuffer, 0, length);
 			termLength = length;
@@ -443,10 +319,9 @@ namespace Lucene.Net.Analysis
 		/// </param>
 		public void  SetTermBuffer(System.String buffer)
 		{
-			termText = null;
 			int length = buffer.Length;
 			GrowTermBuffer(length);
-			SupportClass.TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
+			TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
 			termLength = length;
 		}
 		
@@ -463,9 +338,8 @@ namespace Lucene.Net.Analysis
 		{
 			System.Diagnostics.Debug.Assert(offset <= buffer.Length);
 			System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
-			termText = null;
 			GrowTermBuffer(length);
-			SupportClass.TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
+			TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
 			termLength = length;
 		}
 		
@@ -499,22 +373,7 @@ namespace Lucene.Net.Analysis
 		{
 			if (termBuffer == null)
 			{
-				// The buffer is always at least MIN_BUFFER_SIZE
-				newSize = newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize;
-				//Preserve termText 
-				if (termText != null)
-				{
-					int ttLen = termText.Length;
-					newSize = newSize < ttLen?ttLen:newSize;
-					termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
-					SupportClass.TextSupport.GetCharsFromString(termText, 0, termText.Length, termBuffer, 0);
-					termText = null;
-				}
-				else
-				{
-					// no term Text, the first allocation
-					termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
-				}
+                termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
 			}
 			else
 			{
@@ -553,33 +412,12 @@ namespace Lucene.Net.Analysis
 			}
 		}
 		
-		
-		// TODO: once we remove the deprecated termText() method
-		// and switch entirely to char[] termBuffer we don't need
-		// to use this method anymore, only for late init of the buffer
 		private void  InitTermBuffer()
 		{
 			if (termBuffer == null)
 			{
-				if (termText == null)
-				{
-					termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
-					termLength = 0;
-				}
-				else
-				{
-					int length = termText.Length;
-					if (length < MIN_BUFFER_SIZE)
-						length = MIN_BUFFER_SIZE;
-					termBuffer = new char[ArrayUtil.GetNextSize(length)];
-					termLength = termText.Length;
-					SupportClass.TextSupport.GetCharsFromString(termText, 0, termText.Length, termBuffer, 0);
-					termText = null;
-				}
-			}
-			else
-			{
-				termText = null;
+                termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
+                termLength = 0;
 			}
 		}
 		
@@ -611,7 +449,7 @@ namespace Lucene.Net.Analysis
 		/// <summary>Returns this Token's starting offset, the position of the first character
 		/// corresponding to this token in the source text.
 		/// Note that the difference between endOffset() and startOffset() may not be
-		/// equal to termText.length(), as the term text may have been altered by a
+		/// equal to <see cref="TermLength"/>, as the term text may have been altered by a
 		/// stemmer or some other filter. 
 		/// </summary>
 		public int StartOffset()
@@ -727,7 +565,6 @@ namespace Lucene.Net.Analysis
 			payload = null;
 			// Leave termBuffer to allow re-use
 			termLength = 0;
-			termText = null;
 			positionIncrement = 1;
 			flags = 0;
 			startOffset = endOffset = 0;
@@ -997,11 +834,6 @@ namespace Lucene.Net.Analysis
 				{
 					to.payload = (Payload) payload.Clone();
 				}
-				// remove the following optimization in 3.0 when old TokenStream API removed:
-			}
-			else if (target is TokenWrapper)
-			{
-				((TokenWrapper) target).delegate_Renamed = (Token) this.Clone();
 			}
 			else
 			{
@@ -1014,5 +846,57 @@ namespace Lucene.Net.Analysis
 				((TypeAttribute) target).SetType(type);
 			}
 		}
-	}
+       
+        ///<summary>
+        /// Convenience factory that returns <code>Token</code> as implementation for the basic
+        /// attributes and return the default impl (with &quot;Impl&quot; appended) for all other
+        /// attributes.
+        /// @since 3.0
+        /// </summary>
+	    public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+	        new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+  
+        /// <summary>
+        /// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
+        /// and for all other attributes calls the given delegate factory.
+        /// </summary>
+        public class TokenAttributeFactory : AttributeSource.AttributeFactory
+        {
+
+            private AttributeSource.AttributeFactory _delegateFactory;
+
+            /// <summary>
+            /// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
+            /// and for all other attributes calls the given delegate factory.
+            /// </summary>
+            public TokenAttributeFactory(AttributeSource.AttributeFactory delegateFactory)
+            {
+                this._delegateFactory = delegateFactory;
+            }
+
+            public override AttributeImpl CreateAttributeInstance<T>()
+            {
+                return typeof(T).IsAssignableFrom(typeof(Token))
+                           ? new Token()
+                           : _delegateFactory.CreateAttributeInstance<T>();
+            }
+
+            public override bool Equals(Object other)
+            {
+                if (this == other) return true;
+
+                if (other is TokenAttributeFactory)
+                {
+                    TokenAttributeFactory af = (TokenAttributeFactory)other;
+                    return this._delegateFactory.Equals(af._delegateFactory);
+                }
+                return false;
+            }
+
+            public override int GetHashCode()
+            {
+                return _delegateFactory.GetHashCode() ^ 0x0a45aa31;
+            }
+        }
+    }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/src/core/Analysis/TokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/TokenFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/TokenFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/TokenFilter.cs Tue Feb 28 22:43:08 2012
@@ -31,8 +31,10 @@ namespace Lucene.Net.Analysis
 	{
 		/// <summary>The source of tokens for this filter. </summary>
 		protected internal TokenStream input;
-		
-		/// <summary>Construct a token stream filtering the given input. </summary>
+
+	    private bool isDisposed;
+
+	    /// <summary>Construct a token stream filtering the given input. </summary>
 		protected internal TokenFilter(TokenStream input):base(input)
 		{
 			this.input = input;
@@ -46,12 +48,22 @@ namespace Lucene.Net.Analysis
 		{
 			input.End();
 		}
-		
-		/// <summary>Close the input TokenStream. </summary>
-		public override void  Close()
-		{
-			input.Close();
-		}
+
+        protected override void Dispose(bool disposing)
+        {
+            if (isDisposed) return;
+
+            if (disposing)
+            {
+                if (input != null)
+                {
+                    input.Close();
+                }
+            }
+
+            //input = null;
+            isDisposed = true;
+        }
 		
 		/// <summary>Reset the filter as well as the input TokenStream. </summary>
 		public override void  Reset()

Modified: incubator/lucene.net/trunk/src/core/Analysis/TokenStream.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/TokenStream.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/TokenStream.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/TokenStream.cs Tue Feb 28 22:43:08 2012
@@ -17,12 +17,6 @@
 
 using System;
 
-using FlagsAttribute = Lucene.Net.Analysis.Tokenattributes.FlagsAttribute;
-using OffsetAttribute = Lucene.Net.Analysis.Tokenattributes.OffsetAttribute;
-using PayloadAttribute = Lucene.Net.Analysis.Tokenattributes.PayloadAttribute;
-using PositionIncrementAttribute = Lucene.Net.Analysis.Tokenattributes.PositionIncrementAttribute;
-using TermAttribute = Lucene.Net.Analysis.Tokenattributes.TermAttribute;
-using TypeAttribute = Lucene.Net.Analysis.Tokenattributes.TypeAttribute;
 using Document = Lucene.Net.Documents.Document;
 using Field = Lucene.Net.Documents.Field;
 using IndexWriter = Lucene.Net.Index.IndexWriter;
@@ -82,299 +76,46 @@ namespace Lucene.Net.Analysis
 	/// <see cref="AttributeSource.CaptureState" /> and <see cref="AttributeSource.RestoreState" />
 	/// can be used.
 	/// </summary>
-	public abstract class TokenStream:AttributeSource
+	public abstract class TokenStream : AttributeSource, IDisposable
 	{
-		private void  InitBlock()
-		{
-			supportedMethods = GetSupportedMethods(this.GetType());
-		}
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private static readonly AttributeFactory DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY = new TokenWrapperAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private TokenWrapper tokenWrapper;
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private static bool onlyUseNewAPI = false;
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private MethodSupport supportedMethods;
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private sealed class MethodSupport
-		{
-			internal bool hasIncrementToken;
-			internal bool hasReusableNext;
-			internal bool hasNext;
-			
-			internal MethodSupport(System.Type clazz)
-			{
-				hasIncrementToken = IsMethodOverridden(clazz, "IncrementToken", METHOD_NO_PARAMS);
-				hasReusableNext = IsMethodOverridden(clazz, "Next", METHOD_TOKEN_PARAM);
-				hasNext = IsMethodOverridden(clazz, "Next", METHOD_NO_PARAMS);
-			}
-			
-			private static bool IsMethodOverridden(System.Type clazz, System.String name, System.Type[] params_Renamed)
-			{
-				try
-				{
-					return clazz.GetMethod(name, params_Renamed).DeclaringType != typeof(TokenStream);
-				}
-				catch (System.MethodAccessException e)
-				{
-					// should not happen
-					throw new System.SystemException(e.Message, e);
-				}
-			}
-			
-			private static readonly System.Type[] METHOD_NO_PARAMS = new System.Type[0];
-			private static readonly System.Type[] METHOD_TOKEN_PARAM = new System.Type[]{typeof(Token)};
-		}
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private static readonly System.Collections.Hashtable knownMethodSupport = new System.Collections.Hashtable();
-
-        // {{Aroush-2.9 Port issue, need to mimic java's IdentityHashMap
-        /*
-         * From Java docs:
-         * This class implements the Map interface with a hash table, using 
-         * reference-equality in place of object-equality when comparing keys 
-         * (and values). In other words, in an IdentityHashMap, two keys k1 and k2 
-         * are considered equal if and only if (k1==k2). (In normal Map 
-         * implementations (like HashMap) two keys k1 and k2 are considered 
-         * equal if and only if (k1==null ? k2==null : k1.equals(k2)).) 
-         */
-        // Aroush-2.9}}
-
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private static MethodSupport GetSupportedMethods(System.Type clazz)
-		{
-			MethodSupport supportedMethods;
-			lock (knownMethodSupport)
-			{
-				supportedMethods = (MethodSupport) knownMethodSupport[clazz];
-				if (supportedMethods == null)
-				{
-					knownMethodSupport.Add(clazz, supportedMethods = new MethodSupport(clazz));
-				}
-			}
-			return supportedMethods;
-		}
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private sealed class TokenWrapperAttributeFactory:AttributeFactory
-		{
-			private AttributeFactory delegate_Renamed;
-			
-			internal TokenWrapperAttributeFactory(AttributeFactory delegate_Renamed)
-			{
-				this.delegate_Renamed = delegate_Renamed;
-			}
-			
-			public override AttributeImpl CreateAttributeInstance(System.Type attClass)
-			{
-				return attClass.IsAssignableFrom(typeof(TokenWrapper))?new TokenWrapper():delegate_Renamed.CreateAttributeInstance(attClass);
-			}
-			
-			// this is needed for TeeSinkTokenStream's check for compatibility of AttributeSource,
-			// so two TokenStreams using old API have the same AttributeFactory wrapped by this one.
-			public  override bool Equals(System.Object other)
-			{
-				if (this == other)
-					return true;
-				if (other is TokenWrapperAttributeFactory)
-				{
-					TokenWrapperAttributeFactory af = (TokenWrapperAttributeFactory) other;
-					return this.delegate_Renamed.Equals(af.delegate_Renamed);
-				}
-				return false;
-			}
-			
-			public override int GetHashCode()
-			{
-				return delegate_Renamed.GetHashCode() ^ 0x0a45ff31;
-			}
-		}
-		
 		/// <summary> A TokenStream using the default attribute factory.</summary>
-		protected internal TokenStream():base(onlyUseNewAPI?AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY:TokenStream.DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY)
-		{
-			InitBlock();
-			tokenWrapper = InitTokenWrapper(null);
-			Check();
-		}
+		protected internal TokenStream()
+		{ }
 		
 		/// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
-		protected internal TokenStream(AttributeSource input):base(input)
-		{
-			InitBlock();
-			tokenWrapper = InitTokenWrapper(input);
-			Check();
-		}
+        protected internal TokenStream(AttributeSource input)
+            : base(input)
+		{ }
 		
 		/// <summary> A TokenStream using the supplied AttributeFactory for creating new <see cref="Attribute" /> instances.</summary>
-		protected internal TokenStream(AttributeFactory factory):base(onlyUseNewAPI?factory:new TokenWrapperAttributeFactory(factory))
-		{
-			InitBlock();
-			tokenWrapper = InitTokenWrapper(null);
-			Check();
-		}
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private TokenWrapper InitTokenWrapper(AttributeSource input)
-		{
-			if (onlyUseNewAPI)
-			{
-				// no wrapper needed
-				return null;
-			}
-			else
-			{
-				// if possible get the wrapper from the filter's input stream
-				if (input is TokenStream && ((TokenStream) input).tokenWrapper != null)
-				{
-					return ((TokenStream) input).tokenWrapper;
-				}
-				// check that all attributes are implemented by the same TokenWrapper instance
-				Attribute att = AddAttribute(typeof(TermAttribute));
-				if (att is TokenWrapper && AddAttribute(typeof(TypeAttribute)) == att && AddAttribute(typeof(PositionIncrementAttribute)) == att && AddAttribute(typeof(FlagsAttribute)) == att && AddAttribute(typeof(OffsetAttribute)) == att && AddAttribute(typeof(PayloadAttribute)) == att)
-				{
-					return (TokenWrapper) att;
-				}
-				else
-				{
-					throw new System.NotSupportedException("If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class " + "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been " + "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!");
-				}
-			}
-		}
-		
-		/// <deprecated> Remove this when old API is removed! 
-		/// </deprecated>
-        [Obsolete("Remove this when old API is removed! ")]
-		private void  Check()
-		{
-			if (onlyUseNewAPI && !supportedMethods.hasIncrementToken)
-			{
-				throw new System.NotSupportedException(GetType().FullName + " does not implement incrementToken() which is needed for onlyUseNewAPI.");
-			}
-			
-			// a TokenStream subclass must at least implement one of the methods!
-			if (!(supportedMethods.hasIncrementToken || supportedMethods.hasNext || supportedMethods.hasReusableNext))
-			{
-				throw new System.NotSupportedException(GetType().FullName + " does not implement any of incrementToken(), next(Token), next().");
-			}
-		}
-		
-		/// <summary> For extra performance you can globally enable the new
-		/// <see cref="IncrementToken" /> API using <see cref="Attribute" />s. There will be a
-		/// small, but in most cases negligible performance increase by enabling this,
-		/// but it only works if <b>all</b> <c>TokenStream</c>s use the new API and
-		/// implement <see cref="IncrementToken" />. This setting can only be enabled
-		/// globally.
-		/// <p/>
-		/// This setting only affects <c>TokenStream</c>s instantiated after this
-		/// call. All <c>TokenStream</c>s already created use the other setting.
-		/// <p/>
-		/// All core <see cref="Analyzer" />s are compatible with this setting, if you have
-		/// your own <c>TokenStream</c>s that are also compatible, you should enable
-		/// this.
-		/// <p/>
-		/// When enabled, tokenization may throw <see cref="InvalidOperationException" />
-		/// s, if the whole tokenizer chain is not compatible eg one of the
-		/// <c>TokenStream</c>s does not implement the new <c>TokenStream</c> API.
-		/// <p/>
-		/// The default is <c>false</c>, so there is the fallback to the old API
-		/// available.
-		/// 
-		/// </summary>
-		/// <deprecated> This setting will no longer be needed in Lucene 3.0 as the old
-		/// API will be removed.
-		/// </deprecated>
-        [Obsolete("This setting will no longer be needed in Lucene 3.0 as the old API will be removed.")]
-		public static void  SetOnlyUseNewAPI(bool onlyUseNewAPI)
-		{
-			TokenStream.onlyUseNewAPI = onlyUseNewAPI;
-		}
-		
-		/// <summary> Returns if only the new API is used.
-		/// 
-		/// </summary>
-		/// <seealso cref="SetOnlyUseNewAPI">
-		/// </seealso>
-		/// <deprecated> This setting will no longer be needed in Lucene 3.0 as
-		/// the old API will be removed.
-		/// </deprecated>
-        [Obsolete("This setting will no longer be needed in Lucene 3.0 as the old API will be removed.")]
-		public static bool GetOnlyUseNewAPI()
-		{
-			return onlyUseNewAPI;
-		}
-		
-		/// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
-		/// the next token. Implementing classes must implement this method and update
-		/// the appropriate <see cref="AttributeImpl" />s with the attributes of the next
-		/// token.
-		/// 
-		/// The producer must make no assumptions about the attributes after the
-		/// method has been returned: the caller may arbitrarily change it. If the
-		/// producer needs to preserve the state for subsequent calls, it can use
-		/// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
-		/// 
-		/// This method is called for every token of a document, so an efficient
-		/// implementation is crucial for good performance. To avoid calls to
-		/// <see cref="AttributeSource.AddAttribute(Type)" /> and <see cref="AttributeSource.GetAttribute(Type)" /> or downcasts,
-		/// references to all <see cref="AttributeImpl" />s that this stream uses should be
-		/// retrieved during instantiation.
-		/// 
-		/// To ensure that filters and consumers know which attributes are available,
-		/// the attributes must be added during instantiation. Filters and consumers
-		/// are not required to check for availability of attributes in
-		/// <see cref="IncrementToken()" />.
-		/// 
-		/// </summary>
-		/// <returns> false for end of stream; true otherwise
-		/// 
-		/// Note that this method will be defined abstract in Lucene
-		/// 3.0.
-		/// </returns>
-		public virtual bool IncrementToken()
-		{
-			System.Diagnostics.Debug.Assert(tokenWrapper != null);
-			
-			Token token;
-			if (supportedMethods.hasReusableNext)
-			{
-				token = Next(tokenWrapper.delegate_Renamed);
-			}
-			else
-			{
-				System.Diagnostics.Debug.Assert(supportedMethods.hasNext);
-				token = Next();
-			}
-			if (token == null)
-				return false;
-			tokenWrapper.delegate_Renamed = token;
-			return true;
-		}
+        protected internal TokenStream(AttributeFactory factory)
+            : base(factory)
+		{ }
+
+	    /// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
+	    /// the next token. Implementing classes must implement this method and update
+	    /// the appropriate <see cref="AttributeImpl" />s with the attributes of the next
+	    /// token.
+	    /// 
+	    /// The producer must make no assumptions about the attributes after the
+	    /// method has been returned: the caller may arbitrarily change it. If the
+	    /// producer needs to preserve the state for subsequent calls, it can use
+	    /// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
+	    /// 
+	    /// This method is called for every token of a document, so an efficient
+	    /// implementation is crucial for good performance. To avoid calls to
+	    /// <see cref="AttributeSource.AddAttribute(Type)" /> and <see cref="AttributeSource.GetAttribute(Type)" />,
+	    /// references to all <see cref="AttributeImpl" />s that this stream uses should be
+	    /// retrieved during instantiation.
+	    /// 
+	    /// To ensure that filters and consumers know which attributes are available,
+	    /// the attributes must be added during instantiation. Filters and consumers
+	    /// are not required to check for availability of attributes in
+	    /// <see cref="IncrementToken()" />.
+	    /// 
+	    /// </summary>
+	    /// <returns> false for end of stream; true otherwise</returns>
+	    public abstract bool IncrementToken();
 		
 		/// <summary> This method is called by the consumer after the last token has been
 		/// consumed, after <see cref="IncrementToken" /> returned <c>false</c>
@@ -393,97 +134,6 @@ namespace Lucene.Net.Analysis
 			// do nothing by default
 		}
 		
-		/// <summary> Returns the next token in the stream, or null at EOS. When possible, the
-		/// input Token should be used as the returned Token (this gives fastest
-		/// tokenization performance), but this is not required and a new Token may be
-		/// returned. Callers may re-use a single Token instance for successive calls
-		/// to this method.
-		/// 
-		/// This implicitly defines a "contract" between consumers (callers of this
-		/// method) and producers (implementations of this method that are the source
-		/// for tokens):
-		/// <list type="bullet">
-		/// <item>A consumer must fully consume the previously returned <see cref="Token" />
-		/// before calling this method again.</item>
-		/// <item>A producer must call <see cref="Token.Clear()" /> before setting the fields in
-		/// it and returning it</item>
-		/// </list>
-		/// Also, the producer must make no assumptions about a <see cref="Token" /> after it
-		/// has been returned: the caller may arbitrarily change it. If the producer
-		/// needs to hold onto the <see cref="Token" /> for subsequent calls, it must clone()
-		/// it before storing it. Note that a <see cref="TokenFilter" /> is considered a
-		/// consumer.
-		/// 
-		/// </summary>
-		/// <param name="reusableToken">a <see cref="Token" /> that may or may not be used to return;
-		/// this parameter should never be null (the callee is not required to
-		/// check for null before using it, but it is a good idea to assert that
-		/// it is not null.)
-		/// </param>
-		/// <returns> next <see cref="Token" /> in the stream or null if end-of-stream was hit
-		/// </returns>
-		/// <deprecated> The new <see cref="IncrementToken" /> and <see cref="AttributeSource" />
-		/// APIs should be used instead.
-		/// </deprecated>
-        [Obsolete("The new IncrementToken() and AttributeSource APIs should be used instead.")]
-		public virtual Token Next(Token reusableToken)
-		{
-			System.Diagnostics.Debug.Assert(reusableToken != null);
-			
-			if (tokenWrapper == null)
-				throw new System.NotSupportedException("This TokenStream only supports the new Attributes API.");
-			
-			if (supportedMethods.hasIncrementToken)
-			{
-				tokenWrapper.delegate_Renamed = reusableToken;
-				return IncrementToken()?tokenWrapper.delegate_Renamed:null;
-			}
-			else
-			{
-				System.Diagnostics.Debug.Assert(supportedMethods.hasNext);
-				return Next();
-			}
-		}
-		
-		/// <summary> Returns the next <see cref="Token" /> in the stream, or null at EOS.
-		/// 
-		/// </summary>
-		/// <deprecated> The returned Token is a "full private copy" (not re-used across
-		/// calls to <see cref="Next()" />) but will be slower than calling
-		/// <see cref="Next(Token)" /> or using the new <see cref="IncrementToken()" />
-		/// method with the new <see cref="AttributeSource" /> API.
-		/// </deprecated>
-        [Obsolete("The returned Token is a \"full private copy\" (not re-used across calls to Next()) but will be slower than calling Next(Token) or using the new IncrementToken() method with the new AttributeSource API.")]
-		public virtual Token Next()
-		{
-			if (tokenWrapper == null)
-				throw new System.NotSupportedException("This TokenStream only supports the new Attributes API.");
-			
-			Token nextToken;
-			if (supportedMethods.hasIncrementToken)
-			{
-				Token savedDelegate = tokenWrapper.delegate_Renamed;
-				tokenWrapper.delegate_Renamed = new Token();
-				nextToken = IncrementToken()?tokenWrapper.delegate_Renamed:null;
-				tokenWrapper.delegate_Renamed = savedDelegate;
-			}
-			else
-			{
-				System.Diagnostics.Debug.Assert(supportedMethods.hasReusableNext);
-				nextToken = Next(new Token());
-			}
-			
-			if (nextToken != null)
-			{
-				Lucene.Net.Index.Payload p = nextToken.GetPayload();
-				if (p != null)
-				{
-					nextToken.SetPayload((Lucene.Net.Index.Payload) p.Clone());
-				}
-			}
-			return nextToken;
-		}
-		
 		/// <summary> Resets this stream to the beginning. This is an optional operation, so
 		/// subclasses may or may not implement this method. <see cref="Reset()" /> is not needed for
 		/// the standard indexing process. However, if the tokens of a
@@ -498,8 +148,17 @@ namespace Lucene.Net.Analysis
 		}
 		
 		/// <summary>Releases resources associated with this stream. </summary>
-		public virtual void  Close()
+		[Obsolete("Use Dispose() instead")]
+		public void  Close()
 		{
+            Dispose();
 		}
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+	    protected abstract void Dispose(bool disposing);
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/src/core/Analysis/Tokenattributes/PayloadAttributeImpl.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/Tokenattributes/PayloadAttributeImpl.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/Tokenattributes/PayloadAttributeImpl.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/Tokenattributes/PayloadAttributeImpl.cs Tue Feb 28 22:43:08 2012
@@ -59,9 +59,16 @@ namespace Lucene.Net.Analysis.Tokenattri
 		
 		public override System.Object Clone()
 		{
-            PayloadAttributeImpl impl = new PayloadAttributeImpl();
-            impl.payload = new Payload(this.payload.data, this.payload.offset, this.payload.length);
-            return impl;
+		    var clone = (PayloadAttributeImpl) base.Clone();
+            if (payload != null)
+            {
+                clone.payload = (Payload) payload.Clone();
+            }
+		    return clone;
+            // TODO: This code use to be as below.  Any reason why?  the if(payload!=null) was missing...
+		    //PayloadAttributeImpl impl = new PayloadAttributeImpl();
+		    //impl.payload = new Payload(this.payload.data, this.payload.offset, this.payload.length);
+		    //return impl;
 		}
 		
 		public  override bool Equals(System.Object other)



Mime
View raw message