lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r829134 [1/2] - in /lucene/java/branches/lucene_2_9: ./ contrib/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/...
Date Fri, 23 Oct 2009 17:18:55 GMT
Author: mikemccand
Date: Fri Oct 23 17:18:53 2009
New Revision: 829134

URL: http://svn.apache.org/viewvc?rev=829134&view=rev
Log:
LUCENE-2002 (on 2.9 branch): add Version to QueryParser, MultiFieldQueryParser, StopAnalyzer and the contrib analzyers, to uniformly apply the version-dependent defaults

Modified:
    lucene/java/branches/lucene_2_9/CHANGES.txt
    lucene/java/branches/lucene_2_9/build.xml
    lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java
    lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java
    lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParser.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParser.jj
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
    lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
    lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/queryParser/TestQueryParser.java

Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Fri Oct 23 17:18:53 2009
@@ -42,6 +42,11 @@
    settings in StandardAnalyzer to be compatible with the coming
    Lucene 3.0.  (Uwe Schindler)
    
+ * LUCENE-2002: Add required Version matchVersion argument when
+   constructing QueryParser or MultiFieldQueryParser and, default (as
+   of 2.9) enablePositionIncrements to true to match
+   StandardAnalyzer's 2.9 default (Uwe Schindler, Mike McCandless)
+
 Documentation
 
  * LUCENE-1955: Fix Hits deprecation notice to point users in right

Modified: lucene/java/branches/lucene_2_9/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/build.xml?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/build.xml (original)
+++ lucene/java/branches/lucene_2_9/build.xml Fri Oct 23 17:18:53 2009
@@ -587,9 +587,21 @@
   <target name="javacc" depends="clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser, javacc-contrib-surround"/>
 
   <target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
-    <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
-                   outputDir="src/java/org/apache/lucene/queryParser"
-    />
+    <sequential>
+      <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
+                     outputDir="src/java/org/apache/lucene/queryParser"/>
+
+      <!-- Change the inccorrect public ctors for QueryParser to be protected instead -->
+      <replaceregexp file="src/java/org/apache/lucene/queryParser/QueryParser.java"
+		     byline="true"
+		     match="public QueryParser\(CharStream "
+		     replace="protected QueryParser(CharStream "/>
+      <replaceregexp file="src/java/org/apache/lucene/queryParser/QueryParser.java"
+		     byline="true"
+		     match="public QueryParser\(QueryParserTokenManager "
+		     replace="protected QueryParser(QueryParserTokenManager "/>
+
+    </sequential>
   </target>	
   
   <target name="javacc-HTMLParser" depends="init,javacc-check" if="javacc.present">

Modified: lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/contrib/CHANGES.txt Fri Oct 23 17:18:53 2009
@@ -10,6 +10,15 @@
    list. This has no effect on Arabic text, but if you are using a custom
    stopword list that contains some non-Arabic words, you'll need to fully
    reindex.  (DM Smith via Robert Muir)
+
+API Changes:
+
+ * LUCENE-2002: Add required Version matchVersion argument when
+   constructing ComplexPhraseQueryParser and default (as of 2.9)
+   enablePositionIncrements to true to match StandardAnalyzer's
+   default.  Also added required matchVersion to most of the analyzers
+   (Uwe Schindler, Mike McCandless)
+
       
 Bug fixes
 

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -32,6 +32,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.WordlistLoader;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for Arabic. 
@@ -69,10 +70,22 @@
    */
   public static final String STOPWORDS_COMMENT = "#";
 
+  private final Version matchVersion;
+
   /**
    * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
+   *
+   * @deprecated Use {@link #ArabicAnalyzer(Version)} instead
    */
   public ArabicAnalyzer() {
+    this(Version.LUCENE_24);
+  }
+
+  /**
+   * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
+   */
+  public ArabicAnalyzer(Version matchVersion) {
+    this.matchVersion = matchVersion;
     try {
       InputStream stream = ArabicAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE);
       InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
@@ -87,23 +100,53 @@
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #ArabicAnalyzer(Version, String[])} instead
    */
   public ArabicAnalyzer( String[] stopwords ) {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public ArabicAnalyzer( Version matchVersion, String[] stopwords ) {
     stoptable = StopFilter.makeStopSet( stopwords );
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #ArabicAnalyzer(Version, Hashtable)} instead
    */
   public ArabicAnalyzer( Hashtable stopwords ) {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public ArabicAnalyzer( Version matchVersion, Hashtable stopwords ) {
     stoptable = new HashSet(stopwords.keySet());
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.  Lines can be commented out using {@link #STOPWORDS_COMMENT}
+   *
+   * @deprecated Use {@link #ArabicAnalyzer(Version, File)} instead
    */
   public ArabicAnalyzer( File stopwords ) throws IOException {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.  Lines can be commented out using {@link #STOPWORDS_COMMENT}
+   */
+  public ArabicAnalyzer( Version matchVersion, File stopwords ) throws IOException {
     stoptable = WordlistLoader.getWordSet( stopwords, STOPWORDS_COMMENT);
+    this.matchVersion = matchVersion;
   }
 
 
@@ -117,7 +160,8 @@
   public final TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new ArabicLetterTokenizer( reader );
     result = new LowerCaseFilter(result);
-    result = new StopFilter( result, stoptable );
+    result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                             result, stoptable );
     result = new ArabicNormalizationFilter( result );
     result = new ArabicStemFilter( result );
 
@@ -144,7 +188,8 @@
       streams = new SavedStreams();
       streams.source = new ArabicLetterTokenizer(reader);
       streams.result = new LowerCaseFilter(streams.source);
-      streams.result = new StopFilter(streams.result, stoptable);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stoptable);
       streams.result = new ArabicNormalizationFilter(streams.result);
       streams.result = new ArabicStemFilter(streams.result);
       setPreviousTokenStream(streams);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -32,6 +32,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for Brazilian Portuguese language. 
@@ -40,6 +41,9 @@
  * will not be indexed at all) and an external list of exclusions (words that will
  * not be stemmed, but indexed).
  * </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public final class BrazilianAnalyzer extends Analyzer {
 
@@ -77,33 +81,73 @@
 	 * Contains words that should be indexed but not stemmed.
 	 */
 	private Set excltable = new HashSet();
+        private final Version matchVersion;
 
 	/**
 	 * Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
+         *
+         * @deprecated Use {@link #BrazilianAnalyzer(Version)} instead
 	 */
 	public BrazilianAnalyzer() {
-		stoptable = StopFilter.makeStopSet( BRAZILIAN_STOP_WORDS );
+          this(Version.LUCENE_23);
+	}
+
+	/**
+	 * Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
+	 */
+	public BrazilianAnalyzer(Version matchVersion) {
+          stoptable = StopFilter.makeStopSet( BRAZILIAN_STOP_WORDS );
+          this.matchVersion = matchVersion;
 	}
 
 	/**
 	 * Builds an analyzer with the given stop words.
+         * 
+         * @deprecated Use {@link #BrazilianAnalyzer(Version, String[])} instead
 	 */
 	public BrazilianAnalyzer( String[] stopwords ) {
-		stoptable = StopFilter.makeStopSet( stopwords );
+          this(Version.LUCENE_23, stopwords);
+	}
+
+	/**
+	 * Builds an analyzer with the given stop words.
+	 */
+        public BrazilianAnalyzer( Version matchVersion, String[] stopwords ) {
+          stoptable = StopFilter.makeStopSet( stopwords );
+          this.matchVersion = matchVersion;
 	}
 
 	/**
 	 * Builds an analyzer with the given stop words.
+         *
+         * @deprecated Use {@link #BrazilianAnalyzer(Version, Map)} instead
 	 */
 	public BrazilianAnalyzer( Map stopwords ) {
-		stoptable = new HashSet(stopwords.keySet());
+          this(Version.LUCENE_23, stopwords);
 	}
 
 	/**
 	 * Builds an analyzer with the given stop words.
 	 */
+        public BrazilianAnalyzer( Version matchVersion, Map stopwords ) {
+          stoptable = new HashSet(stopwords.keySet());
+          this.matchVersion = matchVersion;
+	}
+
+	/**
+	 * Builds an analyzer with the given stop words.
+         * @deprecated Use {@link #BrazilianAnalyzer(Version, File)} instead
+	 */
 	public BrazilianAnalyzer( File stopwords ) throws IOException {
-		stoptable = WordlistLoader.getWordSet( stopwords );
+          this(Version.LUCENE_23, stopwords);
+	}
+
+	/**
+	 * Builds an analyzer with the given stop words.
+	 */
+        public BrazilianAnalyzer( Version matchVersion, File stopwords ) throws IOException {
+          stoptable = WordlistLoader.getWordSet( stopwords );
+          this.matchVersion = matchVersion;
 	}
 
 	/**
@@ -136,10 +180,11 @@
 	 *          {@link BrazilianStemFilter}.
 	 */
 	public final TokenStream tokenStream(String fieldName, Reader reader) {
-		TokenStream result = new StandardTokenizer( reader );
+                TokenStream result = new StandardTokenizer( matchVersion, reader );
 		result = new LowerCaseFilter( result );
 		result = new StandardFilter( result );
-		result = new StopFilter( result, stoptable );
+		result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                         result, stoptable );
 		result = new BrazilianStemFilter( result, excltable );
 		return result;
 	}
@@ -162,10 +207,11 @@
       SavedStreams streams = (SavedStreams) getPreviousTokenStream();
       if (streams == null) {
         streams = new SavedStreams();
-        streams.source = new StandardTokenizer(reader);
+        streams.source = new StandardTokenizer(matchVersion, reader);
         streams.result = new LowerCaseFilter(streams.source);
         streams.result = new StandardFilter(streams.result);
-        streams.result = new StopFilter(streams.result, stoptable);
+        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                        streams.result, stoptable);
         streams.result = new BrazilianStemFilter(streams.result, excltable);
         setPreviousTokenStream(streams);
       } else {

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Version;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -56,23 +57,45 @@
    * stop word list
    */
   private Set stopTable;
+  private final Version matchVersion;
 
   //~ Constructors -----------------------------------------------------------
 
   /**
    * Builds an analyzer which removes words in {@link #STOP_WORDS}.
+   *
+   * @deprecated Use {@link #CJKAnalyzer(Version)} instead
    */
   public CJKAnalyzer() {
+    this(Version.LUCENE_24);
+  }
+
+  /**
+   * Builds an analyzer which removes words in {@link #STOP_WORDS}.
+   */
+  public CJKAnalyzer(Version matchVersion) {
     stopTable = StopFilter.makeStopSet(STOP_WORDS);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer which removes words in the provided array.
    *
    * @param stopWords stop word array
+   * @deprecated Use {@link #CJKAnalyzer(Version, String[])} instead
    */
   public CJKAnalyzer(String[] stopWords) {
+    this(Version.LUCENE_24, stopWords);
+  }
+
+  /**
+   * Builds an analyzer which removes words in the provided array.
+   *
+   * @param stopWords stop word array
+   */
+  public CJKAnalyzer(Version matchVersion, String[] stopWords) {
     stopTable = StopFilter.makeStopSet(stopWords);
+    this.matchVersion = matchVersion;
   }
 
   //~ Methods ----------------------------------------------------------------
@@ -86,7 +109,8 @@
    *    {@link StopFilter}
    */
   public final TokenStream tokenStream(String fieldName, Reader reader) {
-    return new StopFilter(new CJKTokenizer(reader), stopTable);
+    return new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                          new CJKTokenizer(reader), stopTable);
   }
   
   private class SavedStreams {
@@ -109,7 +133,8 @@
     if (streams == null) {
       streams = new SavedStreams();
       streams.source = new CJKTokenizer(reader);
-      streams.result = new StopFilter(streams.source, stopTable);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.source, stopTable);
       setPreviousTokenStream(streams);
     } else {
       streams.source.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 import java.io.*;
 import java.util.HashSet;
@@ -37,6 +38,9 @@
  * will not be indexed at all). 
  * A default set of stopwords is used unless an alternative list is specified.
  * </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public final class CzechAnalyzer extends Analyzer {
 
@@ -68,30 +72,68 @@
 	 * Contains the stopwords used with the {@link StopFilter}.
 	 */
 	private Set stoptable;
+        private final Version matchVersion;
 
 	/**
 	 * Builds an analyzer with the default stop words ({@link #CZECH_STOP_WORDS}).
+         *
+         * @deprecated Use {@link #CzechAnalyzer(Version)} instead
 	 */
 	public CzechAnalyzer() {
-		stoptable = StopFilter.makeStopSet( CZECH_STOP_WORDS );
+          this(Version.LUCENE_23);
+	}
+	/**
+	 * Builds an analyzer with the default stop words ({@link #CZECH_STOP_WORDS}).
+	 */
+	public CzechAnalyzer(Version matchVersion) {
+          stoptable = StopFilter.makeStopSet( CZECH_STOP_WORDS );
+          this.matchVersion = matchVersion;
 	}
 
 	/**
 	 * Builds an analyzer with the given stop words.
+         *
+         * @deprecated Use {@link #CzechAnalyzer(Version, String[])} instead
 	 */
 	public CzechAnalyzer( String[] stopwords ) {
-		stoptable = StopFilter.makeStopSet( stopwords );
+          this(Version.LUCENE_23, stopwords);
+	}
+
+	/**
+	 * Builds an analyzer with the given stop words.
+	 */
+        public CzechAnalyzer(Version matchVersion, String[] stopwords) {
+          stoptable = StopFilter.makeStopSet( stopwords );
+          this.matchVersion = matchVersion;
 	}
 
+        /**
+         * @deprecated Use {@link #CzechAnalyzer(Version, HashSet)} instead
+         */
 	public CzechAnalyzer( HashSet stopwords ) {
-		stoptable = stopwords;
+          this(Version.LUCENE_23, stopwords);
+	}
+
+        public CzechAnalyzer(Version matchVersion, HashSet stopwords) {
+          stoptable = stopwords;
+          this.matchVersion = matchVersion;
 	}
 
 	/**
 	 * Builds an analyzer with the given stop words.
+         *
+         * @deprecated Use {@link #CzechAnalyzer(Version, File)} instead
 	 */
 	public CzechAnalyzer( File stopwords ) throws IOException {
-		stoptable = WordlistLoader.getWordSet( stopwords );
+          this(Version.LUCENE_23, stopwords);
+	}
+
+	/**
+	 * Builds an analyzer with the given stop words.
+	 */
+        public CzechAnalyzer(Version matchVersion, File stopwords ) throws IOException {
+          stoptable = WordlistLoader.getWordSet( stopwords );
+          this.matchVersion = matchVersion;
 	}
 
     /**
@@ -135,10 +177,11 @@
 	 * 			{@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
 	 */
 	public final TokenStream tokenStream( String fieldName, Reader reader ) {
-		TokenStream result = new StandardTokenizer( reader );
+                TokenStream result = new StandardTokenizer( matchVersion, reader );
 		result = new StandardFilter( result );
 		result = new LowerCaseFilter( result );
-		result = new StopFilter( result, stoptable );
+		result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                         result, stoptable );
 		return result;
 	}
 	
@@ -159,10 +202,11 @@
       SavedStreams streams = (SavedStreams) getPreviousTokenStream();
       if (streams == null) {
         streams = new SavedStreams();
-        streams.source = new StandardTokenizer(reader);
+        streams.source = new StandardTokenizer(matchVersion, reader);
         streams.result = new StandardFilter(streams.source);
         streams.result = new LowerCaseFilter(streams.result);
-        streams.result = new StopFilter(streams.result, stoptable);
+        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                        streams.result, stoptable);
         setPreviousTokenStream(streams);
       } else {
         streams.source.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -33,6 +33,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for German language. 
@@ -44,6 +45,9 @@
  * exclusion list is empty by default.
  * </p>
  * 
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
+ *
  * @version $Id$
  */
 public class GermanAnalyzer extends Analyzer {
@@ -76,37 +80,80 @@
    */
   private Set exclusionSet = new HashSet();
 
+  private final Version matchVersion;
+
   /**
    * Builds an analyzer with the default stop words:
    * {@link #GERMAN_STOP_WORDS}.
+   *
+   * @deprecated Use {@link #GermanAnalyzer(Version)} instead
    */
   public GermanAnalyzer() {
+    this(Version.LUCENE_23);
+  }
+
+  /**
+   * Builds an analyzer with the default stop words:
+   * {@link #GERMAN_STOP_WORDS}.
+   */
+  public GermanAnalyzer(Version matchVersion) {
     stopSet = StopFilter.makeStopSet(GERMAN_STOP_WORDS);
     setOverridesTokenStreamMethod(GermanAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #GermanAnalyzer(Version, String[])} instead
    */
   public GermanAnalyzer(String[] stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public GermanAnalyzer(Version matchVersion, String[] stopwords) {
     stopSet = StopFilter.makeStopSet(stopwords);
     setOverridesTokenStreamMethod(GermanAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #GermanAnalyzer(Version, Map)} instead
    */
   public GermanAnalyzer(Map stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public GermanAnalyzer(Version matchVersion, Map stopwords) {
     stopSet = new HashSet(stopwords.keySet());
     setOverridesTokenStreamMethod(GermanAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #GermanAnalyzer(Version, File)} instead
    */
   public GermanAnalyzer(File stopwords) throws IOException {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public GermanAnalyzer(Version matchVersion, File stopwords) throws IOException {
     stopSet = WordlistLoader.getWordSet(stopwords);
     setOverridesTokenStreamMethod(GermanAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
   /**
@@ -141,10 +188,11 @@
    *         {@link GermanStemFilter}
    */
   public TokenStream tokenStream(String fieldName, Reader reader) {
-    TokenStream result = new StandardTokenizer(reader);
+    TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
     result = new LowerCaseFilter(result);
-    result = new StopFilter(result, stopSet);
+    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                            result, stopSet);
     result = new GermanStemFilter(result, exclusionSet);
     return result;
   }
@@ -173,10 +221,11 @@
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();
     if (streams == null) {
       streams = new SavedStreams();
-      streams.source = new StandardTokenizer(reader);
+      streams.source = new StandardTokenizer(matchVersion, reader);
       streams.result = new StandardFilter(streams.source);
       streams.result = new LowerCaseFilter(streams.result);
-      streams.result = new StopFilter(streams.result, stopSet);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stopSet);
       streams.result = new GermanStemFilter(streams.result, exclusionSet);
       setPreviousTokenStream(streams);
     } else {

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -36,6 +37,9 @@
  * that will not be indexed at all).
  * A default set of stopwords is used unless an alternative list is specified.
  * </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public final class GreekAnalyzer extends Analyzer
 {
@@ -159,40 +163,62 @@
      */
     private char[] charset;
 
+    private final Version matchVersion;
+
+    /** @deprecated Use {@link #GreekAnalyzer(Version)} instead */
     public GreekAnalyzer() {
+      this(Version.LUCENE_23);
+    }
+
+    public GreekAnalyzer(Version matchVersion) {
         charset = GreekCharsets.UnicodeGreek;
         stopSet = StopFilter.makeStopSet(
                     makeStopWords(GreekCharsets.UnicodeGreek));
+        this.matchVersion = matchVersion;
     }
 
     /**
      * Builds an analyzer.
-     * @deprecated Use {@link #GreekAnalyzer()} instead.
+     * @deprecated Use {@link #GreekAnalyzer(Version)} instead.
      */
     public GreekAnalyzer(char[] charset)
     {
         this.charset = charset;
         stopSet = StopFilter.makeStopSet(makeStopWords(charset));
+        matchVersion = Version.LUCENE_23;
     }
     
     /**
      * Builds an analyzer with the given stop words.
      * @param stopwords Array of stopwords to use.
+     *
+     * @deprecated Use {@link #GreekAnalyzer(Version, String[])} instead
      */
     public GreekAnalyzer(String [] stopwords)
     {
+      this(Version.LUCENE_23, stopwords);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words.
+     * @param stopwords Array of stopwords to use.
+     */
+    public GreekAnalyzer(Version matchVersion, String [] stopwords)
+    {
     	charset = GreekCharsets.UnicodeGreek;
     	stopSet = StopFilter.makeStopSet(stopwords);
+        this.matchVersion = matchVersion;
     }
 
     /**
      * Builds an analyzer with the given stop words.
-     * @deprecated Use {@link #GreekAnalyzer(String[])} instead.
+     * @deprecated Use {@link #GreekAnalyzer(Version, String[])} instead.
      */
     public GreekAnalyzer(char[] charset, String[] stopwords)
     {
         this.charset = charset;
         stopSet = StopFilter.makeStopSet(stopwords);
+        matchVersion = Version.LUCENE_23;
     }
 
     /**
@@ -219,21 +245,33 @@
 
     /**
      * Builds an analyzer with the given stop words.
-     * @deprecated Use {@link #GreekAnalyzer(Map)} instead.
+     * @deprecated Use {@link #GreekAnalyzer(Version, Map)} instead.
      */
     public GreekAnalyzer(char[] charset, Map stopwords)
     {
         this.charset = charset;
         stopSet = new HashSet(stopwords.keySet());
+        matchVersion = Version.LUCENE_23;
     }
     
     /**
      * Builds an analyzer with the given stop words.
+     *
+     * @deprecated Use {@link #GreekAnalyzer(Version,Map)} instead
      */
     public GreekAnalyzer(Map stopwords)
     {
+      this(Version.LUCENE_23, stopwords);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words.
+     */
+    public GreekAnalyzer(Version matchVersion, Map stopwords)
+    {
     	charset = GreekCharsets.UnicodeGreek;
     	stopSet = new HashSet(stopwords.keySet());
+        this.matchVersion = matchVersion;
     }
 
     /**
@@ -244,9 +282,10 @@
      */
     public TokenStream tokenStream(String fieldName, Reader reader)
     {
-    	TokenStream result = new StandardTokenizer(reader);
+        TokenStream result = new StandardTokenizer(matchVersion, reader);
         result = new GreekLowerCaseFilter(result, charset);
-        result = new StopFilter(result, stopSet);
+        result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                result, stopSet);
         return result;
     }
     
@@ -267,9 +306,10 @@
       SavedStreams streams = (SavedStreams) getPreviousTokenStream();
       if (streams == null) {
         streams = new SavedStreams();
-        streams.source = new StandardTokenizer(reader);
+        streams.source = new StandardTokenizer(matchVersion, reader);
         streams.result = new GreekLowerCaseFilter(streams.source, charset);
-        streams.result = new StopFilter(streams.result, stopSet);
+        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                        streams.result, stopSet);
         setPreviousTokenStream(streams);
       } else {
         streams.source.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -34,6 +34,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.ar.ArabicLetterTokenizer;
 import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for Persian.
@@ -66,11 +67,24 @@
    */
   public static final String STOPWORDS_COMMENT = "#";
 
+  private final Version matchVersion;
+
   /**
    * Builds an analyzer with the default stop words:
    * {@link #DEFAULT_STOPWORD_FILE}.
+   *
+   * @deprecated Use {@link #PersianAnalyzer(Version)} instead
    */
   public PersianAnalyzer() {
+    this(Version.LUCENE_24);
+  }
+
+  /**
+   * Builds an analyzer with the default stop words:
+   * {@link #DEFAULT_STOPWORD_FILE}.
+   */
+  public PersianAnalyzer(Version matchVersion) {
+    this.matchVersion = matchVersion;
     try {
       InputStream stream = PersianAnalyzer.class
           .getResourceAsStream(DEFAULT_STOPWORD_FILE);
@@ -86,24 +100,55 @@
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #PersianAnalyzer(Version, String[])} instead
    */
   public PersianAnalyzer(String[] stopwords) {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public PersianAnalyzer(Version matchVersion, String[] stopwords) {
     stoptable = StopFilter.makeStopSet(stopwords);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #PersianAnalyzer(Version, Hashtable)} instead
    */
   public PersianAnalyzer(Hashtable stopwords) {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public PersianAnalyzer(Version matchVersion, Hashtable stopwords) {
     stoptable = new HashSet(stopwords.keySet());
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words. Lines can be commented out
    * using {@link #STOPWORDS_COMMENT}
+   *
+   * @deprecated Use {@link #PersianAnalyzer(Version, File)} instead
    */
   public PersianAnalyzer(File stopwords) throws IOException {
+    this(Version.LUCENE_24, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words. Lines can be commented out
+   * using {@link #STOPWORDS_COMMENT}
+   */
+  public PersianAnalyzer(Version matchVersion, File stopwords) throws IOException {
     stoptable = WordlistLoader.getWordSet(stopwords, STOPWORDS_COMMENT);
+    this.matchVersion = matchVersion;
   }
 
   /**
@@ -125,7 +170,8 @@
      * the order here is important: the stopword list is normalized with the
      * above!
      */
-    result = new StopFilter(result, stoptable);
+    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                            result, stoptable);
 
     return result;
   }
@@ -158,7 +204,8 @@
        * the order here is important: the stopword list is normalized with the
        * above!
        */
-      streams.result = new StopFilter(streams.result, stoptable);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stoptable);
       setPreviousTokenStream(streams);
     } else {
       streams.source.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 import java.io.File;
 import java.io.IOException;
@@ -43,6 +44,17 @@
  * exclusion list is empty by default.
  * </p>
  *
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating FrenchAnalyzer:
+ * <ul>
+ *   <li> As of 2.9, StopFilter preserves position
+ *        increments
+ * </ul>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
+ *
  * @version $Id$
  */
 public final class FrenchAnalyzer extends Analyzer {
@@ -84,26 +96,60 @@
    */
   private Set excltable = new HashSet();
 
+  private final Version matchVersion;
+
   /**
    * Builds an analyzer with the default stop words ({@link #FRENCH_STOP_WORDS}).
+   *
+   * @deprecated Use {@link #FrenchAnalyzer(Version)} instead.
    */
   public FrenchAnalyzer() {
+    this(Version.LUCENE_23);
+  }
+
+  /**
+   * Builds an analyzer with the default stop words ({@link #FRENCH_STOP_WORDS}).
+   */
+  public FrenchAnalyzer(Version matchVersion) {
     stoptable = StopFilter.makeStopSet(FRENCH_STOP_WORDS);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
+   *
+   * @deprecated Use {@link #FrenchAnalyzer(Version,
+   * String[])} instead.
    */
   public FrenchAnalyzer(String[] stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   */
+  public FrenchAnalyzer(Version matchVersion, String[] stopwords) {
     stoptable = StopFilter.makeStopSet(stopwords);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
    * @throws IOException
+   *
+   * @deprecated Use {@link #FrenchAnalyzer(Version, File)} instead
    */
   public FrenchAnalyzer(File stopwords) throws IOException {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   * @throws IOException
+   */
+  public FrenchAnalyzer(Version matchVersion, File stopwords) throws IOException {
     stoptable = new HashSet(WordlistLoader.getWordSet(stopwords));
+    this.matchVersion = matchVersion;
   }
 
   /**
@@ -144,9 +190,10 @@
     if (fieldName == null) throw new IllegalArgumentException("fieldName must not be null");
     if (reader == null) throw new IllegalArgumentException("reader must not be null");
 
-    TokenStream result = new StandardTokenizer(reader);
+    TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
-    result = new StopFilter(result, stoptable);
+    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                            result, stoptable);
     result = new FrenchStemFilter(result, excltable);
     // Convert to lowercase after stemming!
     result = new LowerCaseFilter(result);
@@ -171,9 +218,10 @@
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();
     if (streams == null) {
       streams = new SavedStreams();
-      streams.source = new StandardTokenizer(reader);
+      streams.source = new StandardTokenizer(matchVersion, reader);
       streams.result = new StandardFilter(streams.source);
-      streams.result = new StopFilter(streams.result, stoptable);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stoptable);
       streams.result = new FrenchStemFilter(streams.result, excltable);
       // Convert to lowercase after stemming!
       streams.result = new LowerCaseFilter(streams.result);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 import java.io.File;
 import java.io.IOException;
@@ -42,6 +43,9 @@
  * A default set of stopwords is used unless an alternative list is specified, but the
  * exclusion list is empty by default.
  * </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public class DutchAnalyzer extends Analyzer {
   /**
@@ -73,48 +77,92 @@
   private Set excltable = new HashSet();
 
   private Map stemdict = new HashMap();
-
+  private final Version matchVersion;
 
   /**
    * Builds an analyzer with the default stop words ({@link #DUTCH_STOP_WORDS}) 
    * and a few default entries for the stem exclusion table.
    * 
+   * @deprecated Use {@link #DutchAnalyzer(Version)} instead
    */
   public DutchAnalyzer() {
+    this(Version.LUCENE_23);
+  }
+
+  /**
+   * Builds an analyzer with the default stop words ({@link #DUTCH_STOP_WORDS}) 
+   * and a few default entries for the stem exclusion table.
+   * 
+   */
+  public DutchAnalyzer(Version matchVersion) {
     setOverridesTokenStreamMethod(DutchAnalyzer.class);
     stoptable = StopFilter.makeStopSet(DUTCH_STOP_WORDS);
     stemdict.put("fiets", "fiets"); //otherwise fiet
     stemdict.put("bromfiets", "bromfiets"); //otherwise bromfiet
     stemdict.put("ei", "eier");
     stemdict.put("kind", "kinder");
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
    *
    * @param stopwords
+   * @deprecated Use {@link #DutchAnalyzer(Version, String[])} instead
    */
   public DutchAnalyzer(String[] stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   *
+   * @param matchVersion
+   * @param stopwords
+   */
+  public DutchAnalyzer(Version matchVersion, String[] stopwords) {
     setOverridesTokenStreamMethod(DutchAnalyzer.class);
     stoptable = StopFilter.makeStopSet(stopwords);
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
    *
    * @param stopwords
+   * @deprecated Use {@link #DutchAnalyzer(Version, HashSet)} instead
    */
   public DutchAnalyzer(HashSet stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   *
+   * @param stopwords
+   */
+  public DutchAnalyzer(Version matchVersion, HashSet stopwords) {
     setOverridesTokenStreamMethod(DutchAnalyzer.class);
     stoptable = stopwords;
+    this.matchVersion = matchVersion;
   }
 
   /**
    * Builds an analyzer with the given stop words.
    *
    * @param stopwords
+   * @deprecated Use {@link #DutchAnalyzer(Version, File)} instead
    */
   public DutchAnalyzer(File stopwords) {
+    this(Version.LUCENE_23, stopwords);
+  }
+
+  /**
+   * Builds an analyzer with the given stop words.
+   *
+   * @param stopwords
+   */
+  public DutchAnalyzer(Version matchVersion, File stopwords) {
     setOverridesTokenStreamMethod(DutchAnalyzer.class);
     try {
       stoptable = org.apache.lucene.analysis.WordlistLoader.getWordSet(stopwords);
@@ -122,6 +170,7 @@
       // TODO: throw IOException
       throw new RuntimeException(e);
     }
+    this.matchVersion = matchVersion;
   }
 
   /**
@@ -179,9 +228,10 @@
    *   and {@link DutchStemFilter}
    */
   public TokenStream tokenStream(String fieldName, Reader reader) {
-    TokenStream result = new StandardTokenizer(reader);
+    TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
-    result = new StopFilter(result, stoptable);
+    result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                            result, stoptable);
     result = new DutchStemFilter(result, excltable, stemdict);
     return result;
   }
@@ -211,9 +261,10 @@
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();
     if (streams == null) {
       streams = new SavedStreams();
-      streams.source = new StandardTokenizer(reader);
+      streams.source = new StandardTokenizer(matchVersion, reader);
       streams.result = new StandardFilter(streams.source);
-      streams.result = new StopFilter(streams.result, stoptable);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stoptable);
       streams.result = new DutchStemFilter(streams.result, excltable, stemdict);
       setPreviousTokenStream(streams);
     } else {

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,6 +23,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Version;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -48,15 +49,27 @@
   //The default maximum percentage (40%) of index documents which
   //can contain a term, after which the term is considered to be a stop word.
   public static final float defaultMaxDocFreqPercent = 0.4f;
+  private final Version matchVersion;
 
   /**
    * Initializes this analyzer with the Analyzer object that actually produces the tokens
    *
    * @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
+   * @deprecated Use {@link #QueryAutoStopWordAnalyzer(Version, Analyzer)} instead
    */
   public QueryAutoStopWordAnalyzer(Analyzer delegate) {
+    this(Version.LUCENE_24, delegate);
+  }
+
+  /**
+   * Initializes this analyzer with the Analyzer object that actually produces the tokens
+   *
+   * @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
+   */
+  public QueryAutoStopWordAnalyzer(Version matchVersion, Analyzer delegate) {
     this.delegate = delegate;
     setOverridesTokenStreamMethod(QueryAutoStopWordAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
   /**
@@ -175,7 +188,8 @@
     }
     HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
     if (stopWords != null) {
-      result = new StopFilter(result, stopWords);
+      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                              result, stopWords);
     }
     return result;
   }
@@ -217,7 +231,8 @@
       /* if there are any stopwords for the field, save the stopfilter */
       HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
       if (stopWords != null)
-        streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
+        streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                                streams.wrapped, stopWords);
       else
         streams.withStopFilter = streams.wrapped;
 
@@ -238,7 +253,8 @@
         streams.wrapped = result;
         HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
         if (stopWords != null)
-          streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
+          streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                                  streams.wrapped, stopWords);
         else
           streams.withStopFilter = streams.wrapped;
       }

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -27,6 +27,7 @@
 import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for Russian language. 
@@ -193,41 +194,60 @@
      * @deprecated Support for non-Unicode encodings will be removed in Lucene 3.0
      */
     private char[] charset;
+    private final Version matchVersion;
 
-
+    /** @deprecated Use {@link #RussianAnalyzer(Version)} instead */
     public RussianAnalyzer() {
+      this(Version.LUCENE_24);
+    }
+
+    public RussianAnalyzer(Version matchVersion) {
         charset = RussianCharsets.UnicodeRussian;
         stopSet = StopFilter.makeStopSet(
                     makeStopWords(RussianCharsets.UnicodeRussian));
+        this.matchVersion = matchVersion;
     }
 
     /**
      * Builds an analyzer.
-     * @deprecated Use {@link #RussianAnalyzer()} instead.
+     * @deprecated Use {@link #RussianAnalyzer(Version)} instead.
      */
     public RussianAnalyzer(char[] charset)
     {
         this.charset = charset;
         stopSet = StopFilter.makeStopSet(makeStopWords(charset));
+        matchVersion = Version.LUCENE_24;
     }
 
     /**
      * Builds an analyzer with the given stop words.
-     * @deprecated Use {@link #RussianAnalyzer(String[])} instead.
+     * @deprecated Use {@link #RussianAnalyzer(Version,String[])} instead.
      */
     public RussianAnalyzer(char[] charset, String[] stopwords)
     {
         this.charset = charset;
         stopSet = StopFilter.makeStopSet(stopwords);
+        matchVersion = Version.LUCENE_24;
     }
     
     /**
      * Builds an analyzer with the given stop words.
+     *
+     * @deprecated Use {@link #RussianAnalyzer(Version,String[])} instead.
      */
     public RussianAnalyzer(String[] stopwords)
     {
+      this(Version.LUCENE_24, stopwords);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words.
+     */
+    public RussianAnalyzer(Version matchVersion, String[] stopwords)
+    {
     	this.charset = RussianCharsets.UnicodeRussian;
     	stopSet = StopFilter.makeStopSet(stopwords);
+        this.matchVersion = matchVersion;
     }
 
     /** Takes russian stop words and translates them to a String array, using
@@ -254,22 +274,36 @@
     /**
      * Builds an analyzer with the given stop words.
      * TODO: create a Set version of this ctor
-     * @deprecated Use {@link #RussianAnalyzer(Map)} instead.
+     *
+     * @deprecated Use {@link #RussianAnalyzer(Version, Map)} instead.
      */
     public RussianAnalyzer(char[] charset, Map stopwords)
     {
         this.charset = charset;
         stopSet = new HashSet(stopwords.keySet());
+        matchVersion = Version.LUCENE_24;
     }
     
     /**
      * Builds an analyzer with the given stop words.
      * TODO: create a Set version of this ctor
+     *
+     * @deprecated Use {@link #RussianAnalyzer(Version, Map)} instead.
      */
     public RussianAnalyzer(Map stopwords)
     {
+      this(Version.LUCENE_24, stopwords);
+    }
+
+    /**
+     * Builds an analyzer with the given stop words.
+     * TODO: create a Set version of this ctor
+     */
+    public RussianAnalyzer(Version matchVersion, Map stopwords)
+    {
     	charset = RussianCharsets.UnicodeRussian;
     	stopSet = new HashSet(stopwords.keySet());
+        this.matchVersion = matchVersion;
     }
 
     /**
@@ -285,7 +319,8 @@
     {
         TokenStream result = new RussianLetterTokenizer(reader, charset);
         result = new RussianLowerCaseFilter(result, charset);
-        result = new StopFilter(result, stopSet);
+        result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                result, stopSet);
         result = new RussianStemFilter(result, charset);
         return result;
     }
@@ -311,7 +346,8 @@
       streams = new SavedStreams();
       streams.source = new RussianLetterTokenizer(reader, charset);
       streams.result = new RussianLowerCaseFilter(streams.source, charset);
-      streams.result = new StopFilter(streams.result, stopSet);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, stopSet);
       streams.result = new RussianStemFilter(streams.result, charset);
       setPreviousTokenStream(streams);
     } else {

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,22 +25,34 @@
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
 
 /**
  * {@link Analyzer} for Thai language. It uses {@link java.text.BreakIterator} to break words.
  * @version 0.2
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public class ThaiAnalyzer extends Analyzer {
-  
+  private final Version matchVersion;
+
+  /** @deprecated Use {@link #ThaiAnalyzer(Version)} instead */
   public ThaiAnalyzer() {
+    this(Version.LUCENE_23);
+  }
+  
+  public ThaiAnalyzer(Version matchVersion) {
     setOverridesTokenStreamMethod(ThaiAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
   
   public TokenStream tokenStream(String fieldName, Reader reader) {
-	  TokenStream ts = new StandardTokenizer(reader);
+    TokenStream ts = new StandardTokenizer(matchVersion, reader);
     ts = new StandardFilter(ts);
     ts = new ThaiWordFilter(ts);
-    ts = new StopFilter(ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+    ts = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                        ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
     return ts;
   }
   
@@ -60,10 +72,11 @@
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();
     if (streams == null) {
       streams = new SavedStreams();
-      streams.source = new StandardTokenizer(reader);
+      streams.source = new StandardTokenizer(matchVersion, reader);
       streams.result = new StandardFilter(streams.source);
       streams.result = new ThaiWordFilter(streams.result);
-      streams.result = new StopFilter(streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+      streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                      streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
       setPreviousTokenStream(streams);
     } else {
       streams.source.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Version;
 
 /**
  * Test case for FrenchAnalyzer.
@@ -32,7 +33,7 @@
 public class TestFrenchAnalyzer extends BaseTokenStreamTestCase {
 
 	public void testAnalyzer() throws Exception {
-		FrenchAnalyzer fa = new FrenchAnalyzer();
+		FrenchAnalyzer fa = new FrenchAnalyzer(Version.LUCENE_CURRENT);
 	
 		// test null reader
 		boolean iaeFlag = false;

Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -31,6 +31,7 @@
 import org.apache.lucene.analysis.WordlistLoader;
 import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
 import org.apache.lucene.analysis.cn.smart.WordTokenFilter;
+import org.apache.lucene.util.Version;
 
 /**
  * <p>
@@ -59,12 +60,22 @@
 public class SmartChineseAnalyzer extends Analyzer {
 
   private final Set stopWords;
+  private final Version matchVersion;
 
   /**
    * Create a new SmartChineseAnalyzer, using the default stopword list.
+   *
+   * @deprecated Use {@link #SmartChineseAnalyzer(Version)} instead
    */
   public SmartChineseAnalyzer() {
-    this(true);
+    this(Version.LUCENE_24, true);
+  }
+
+  /**
+   * Create a new SmartChineseAnalyzer, using the default stopword list.
+   */
+  public SmartChineseAnalyzer(Version matchVersion) {
+    this(matchVersion, true);
   }
 
   /**
@@ -77,8 +88,26 @@
    * </p>
    * 
    * @param useDefaultStopWords true to use the default stopword list.
+   *
+   * @deprecated Use {@link #SmartChineseAnalyzer(Version, boolean)} instead
    */
   public SmartChineseAnalyzer(boolean useDefaultStopWords) {
+    this(Version.LUCENE_24, useDefaultStopWords);
+  }
+
+  /**
+   * <p>
+   * Create a new SmartChineseAnalyzer, optionally using the default stopword list.
+   * </p>
+   * <p>
+   * The included default stopword list is simply a list of punctuation.
+   * If you do not use this list, punctuation will not be removed from the text!
+   * </p>
+   * 
+   * @param useDefaultStopWords true to use the default stopword list.
+   */
+  public SmartChineseAnalyzer(Version matchVersion, boolean useDefaultStopWords) {
+    this.matchVersion = matchVersion;
     if (useDefaultStopWords) {
       try {
       InputStream stream = this.getClass().getResourceAsStream("stopwords.txt");
@@ -101,9 +130,25 @@
    * Note: the set should include punctuation, unless you want to index punctuation!
    * </p>
    * @param stopWords {@link Set} of stopwords to use.
+   *
+   * @deprecated Use {@link #SmartChineseAnalyzer(Version, Set)} instead
    */
   public SmartChineseAnalyzer(Set stopWords) {
+    this(Version.LUCENE_24, stopWords);
+  }
+
+  /**
+   * <p>
+   * Create a new SmartChineseAnalyzer, using the provided {@link Set} of stopwords.
+   * </p>
+   * <p>
+   * Note: the set should include punctuation, unless you want to index punctuation!
+   * </p>
+   * @param stopWords {@link Set} of stopwords to use.
+   */
+  public SmartChineseAnalyzer(Version matchVersion, Set stopWords) {
     this.stopWords = stopWords;
+    this.matchVersion = matchVersion;
   }
 
   public TokenStream tokenStream(String fieldName, Reader reader) {
@@ -114,7 +159,8 @@
     // The porter stemming is too strict, this is not a bug, this is a feature:)
     result = new PorterStemFilter(result);
     if (stopWords != null) {
-      result = new StopFilter(result, stopWords, false);
+      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                              result, stopWords, false);
     }
     return result;
   }
@@ -134,7 +180,8 @@
       streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
       streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
       if (stopWords != null) {
-        streams.filteredTokenStream = new StopFilter(streams.filteredTokenStream, stopWords, false);
+        streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                                     streams.filteredTokenStream, stopWords, false);
       }
     } else {
       streams.tokenStream.reset(reader);

Modified: lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -33,6 +33,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.util.Version;
 
 /**
  * Efficient Lucene analyzer/tokenizer that preferably operates on a String rather than a
@@ -139,6 +140,8 @@
   private final Pattern pattern;
   private final boolean toLowerCase;
   private final Set stopWords;
+
+  private final Version matchVersion;
   
   /**
    * Constructs a new instance with the given parameters.
@@ -157,8 +160,33 @@
    *            <code>WordlistLoader.getWordSet(new File("samples/fulltext/stopwords.txt")</code>
    *            or <a href="http://www.unine.ch/info/clef/">other stop words
    *            lists </a>.
+   *
+   * @deprecated Use {@link #PatternAnalyzer(Version, Pattern, boolean, Set)} instead
    */
   public PatternAnalyzer(Pattern pattern, boolean toLowerCase, Set stopWords) {
+    this(Version.LUCENE_24, pattern, toLowerCase, stopWords);
+  }
+
+  /**
+   * Constructs a new instance with the given parameters.
+   * 
+   * @param matchVersion If >= {@link Version#LUCENE_29}, StopFilter.enablePositionIncrement is set to true
+   * @param pattern
+   *            a regular expression delimiting tokens
+   * @param toLowerCase
+   *            if <code>true</code> returns tokens after applying
+   *            String.toLowerCase()
+   * @param stopWords
+   *            if non-null, ignores all tokens that are contained in the
+   *            given stop set (after previously having applied toLowerCase()
+   *            if applicable). For example, created via
+   *            {@link StopFilter#makeStopSet(String[])}and/or
+   *            {@link org.apache.lucene.analysis.WordlistLoader}as in
+   *            <code>WordlistLoader.getWordSet(new File("samples/fulltext/stopwords.txt")</code>
+   *            or <a href="http://www.unine.ch/info/clef/">other stop words
+   *            lists </a>.
+   */
+  public PatternAnalyzer(Version matchVersion, Pattern pattern, boolean toLowerCase, Set stopWords) {
     if (pattern == null) 
       throw new IllegalArgumentException("pattern must not be null");
     
@@ -170,6 +198,7 @@
     this.pattern = pattern;
     this.toLowerCase = toLowerCase;
     this.stopWords = stopWords;
+    this.matchVersion = matchVersion;
   }
   
   /**
@@ -197,7 +226,7 @@
     }
     else {
       stream = new PatternTokenizer(text, pattern, toLowerCase);
-      if (stopWords != null) stream = new StopFilter(false, stream, stopWords);
+      if (stopWords != null) stream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), stream, stopWords);
     }
     
     return stream;

Modified: lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java Fri Oct 23 17:18:53 2009
@@ -38,6 +38,7 @@
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.Version;
 
 /**
  * QueryParser which permits complex phrase query syntax eg "(john jon
@@ -67,8 +68,15 @@
 
   private ComplexPhraseQuery currentPhraseQuery = null;
 
+  /** @deprecated Use {@link
+  #ComplexPhraseQueryParser{Version, String, Analyzer)}
+  instead.*/
   public ComplexPhraseQueryParser(String f, Analyzer a) {
-    super(f, a);
+    this(Version.LUCENE_24, f, a);
+  }
+
+  public ComplexPhraseQueryParser(Version matchVersion, String f, Analyzer a) {
+    super(matchVersion, f, a);
   }
 
   protected Query getFieldQuery(String field, String queryText, int slop) {

Modified: lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.*;
 import org.apache.lucene.analysis.standard.*;
+import org.apache.lucene.util.Version;
 
 import java.io.IOException;
 import java.io.Reader;
@@ -30,20 +31,39 @@
  * Available stemmers are listed in org.tartarus.snowball.ext.  The name of a
  * stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
  * {@link org.tartarus.snowball.ext.EnglishStemmer} is named "English".
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
  */
 public class SnowballAnalyzer extends Analyzer {
   private String name;
   private Set stopSet;
+  private final Version matchVersion;
 
-  /** Builds the named analyzer with no stop words. */
+  /** Builds the named analyzer with no stop words.
+   *
+   * @deprecated Use {@link {#SnowballAnalyzer(Version, String)} instead*/
   public SnowballAnalyzer(String name) {
+    this(Version.LUCENE_23, name);
+  }
+
+  /** Builds the named analyzer with no stop words. */
+  public SnowballAnalyzer(Version matchVersion, String name) {
     this.name = name;
     setOverridesTokenStreamMethod(SnowballAnalyzer.class);
+    this.matchVersion = matchVersion;
   }
 
-  /** Builds the named analyzer with the given stop words. */
+  /** Builds the named analyzer with the given stop words.
+   *
+   * @deprecated Use {@link {#SnowballAnalyzer(Version, String, String[])} instead*/
   public SnowballAnalyzer(String name, String[] stopWords) {
-    this(name);
+    this(Version.LUCENE_23, name, stopWords);
+  }
+
+  /** Builds the named analyzer with the given stop words. */
+  public SnowballAnalyzer(Version matchVersion, String name, String[] stopWords) {
+    this(matchVersion, name);
     stopSet = StopFilter.makeStopSet(stopWords);
   }
 
@@ -51,11 +71,12 @@
       StandardFilter}, a {@link LowerCaseFilter}, a {@link StopFilter},
       and a {@link SnowballFilter} */
   public TokenStream tokenStream(String fieldName, Reader reader) {
-    TokenStream result = new StandardTokenizer(reader);
+    TokenStream result = new StandardTokenizer(matchVersion, reader);
     result = new StandardFilter(result);
     result = new LowerCaseFilter(result);
     if (stopSet != null)
-      result = new StopFilter(result, stopSet);
+      result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                              result, stopSet);
     result = new SnowballFilter(result, name);
     return result;
   }
@@ -80,11 +101,12 @@
     SavedStreams streams = (SavedStreams) getPreviousTokenStream();
     if (streams == null) {
       streams = new SavedStreams();
-      streams.source = new StandardTokenizer(reader);
+      streams.source = new StandardTokenizer(matchVersion, reader);
       streams.result = new StandardFilter(streams.source);
       streams.result = new LowerCaseFilter(streams.result);
       if (stopSet != null)
-        streams.result = new StopFilter(streams.result, stopSet);
+        streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+                                        streams.result, stopSet);
       streams.result = new SnowballFilter(streams.result, name);
       setPreviousTokenStream(streams);
     } else {

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,7 +23,18 @@
 import java.util.Arrays;
 import java.util.Set;
 
-/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}. */
+import org.apache.lucene.util.Version;
+
+/** Filters {@link LetterTokenizer} with {@link
+ * LowerCaseFilter} and {@link StopFilter}.
+ *
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating StopAnalyzer:
+ * <ul>
+ *   <li> As of 2.9, position increments are preserved
+ * </ul>
+*/
 
 public final class StopAnalyzer extends Analyzer {
   private final Set/*<String>*/ stopWords;
@@ -61,7 +72,7 @@
   
   /** Builds an analyzer which removes words in
    * ENGLISH_STOP_WORDS.
-   * @deprecated Use {@link #StopAnalyzer(boolean)} instead */
+   * @deprecated Use {@link #StopAnalyzer(Version)} instead */
   public StopAnalyzer() {
     stopWords = ENGLISH_STOP_WORDS_SET;
     useDefaultStopPositionIncrement = true;
@@ -69,9 +80,18 @@
   }
 
   /** Builds an analyzer which removes words in
+   * ENGLISH_STOP_WORDS.*/
+  public StopAnalyzer(Version matchVersion) {
+    stopWords = ENGLISH_STOP_WORDS_SET;
+    useDefaultStopPositionIncrement = false;
+    enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+  }
+
+  /** Builds an analyzer which removes words in
    *  ENGLISH_STOP_WORDS.
    * @param enablePositionIncrements See {@link
-   * StopFilter#setEnablePositionIncrements} */
+   * StopFilter#setEnablePositionIncrements} 
+   * @deprecated Use {@link #StopAnalyzer(Version)} instead */
   public StopAnalyzer(boolean enablePositionIncrements) {
     stopWords = ENGLISH_STOP_WORDS_SET;
     this.enablePositionIncrements = enablePositionIncrements;
@@ -79,17 +99,26 @@
   }
 
   /** Builds an analyzer with the stop words from the given set.
-   * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead */
+   * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
   public StopAnalyzer(Set stopWords) {
     this.stopWords = stopWords;
     useDefaultStopPositionIncrement = true;
     enablePositionIncrements = false;
   }
 
+  /** Builds an analyzer with the stop words from the given
+   * set. */
+  public StopAnalyzer(Version matchVersion, Set stopWords) {
+    this.stopWords = stopWords;
+    useDefaultStopPositionIncrement = false;
+    enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+  }
+
   /** Builds an analyzer with the stop words from the given set.
    * @param stopWords Set of stop words
    * @param enablePositionIncrements See {@link
-   * StopFilter#setEnablePositionIncrements} */
+   * StopFilter#setEnablePositionIncrements}
+   * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
   public StopAnalyzer(Set stopWords, boolean enablePositionIncrements) {
     this.stopWords = stopWords;
     this.enablePositionIncrements = enablePositionIncrements;
@@ -97,7 +126,7 @@
   }
 
   /** Builds an analyzer which removes words in the provided array.
-   * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead */
+   * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
   public StopAnalyzer(String[] stopWords) {
     this.stopWords = StopFilter.makeStopSet(stopWords);
     useDefaultStopPositionIncrement = true;
@@ -108,7 +137,7 @@
    * @param stopWords Array of stop words
    * @param enablePositionIncrements See {@link
    * StopFilter#setEnablePositionIncrements} 
-   * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead*/
+   * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead*/
   public StopAnalyzer(String[] stopWords, boolean enablePositionIncrements) {
     this.stopWords = StopFilter.makeStopSet(stopWords);
     this.enablePositionIncrements = enablePositionIncrements;
@@ -117,7 +146,7 @@
   
   /** Builds an analyzer with the stop words from the given file.
    * @see WordlistLoader#getWordSet(File)
-   * @deprecated Use {@link #StopAnalyzer(File, boolean)} instead */
+   * @deprecated Use {@link #StopAnalyzer(Version, File)} instead */
   public StopAnalyzer(File stopwordsFile) throws IOException {
     stopWords = WordlistLoader.getWordSet(stopwordsFile);
     useDefaultStopPositionIncrement = true;
@@ -128,16 +157,27 @@
    * @see WordlistLoader#getWordSet(File)
    * @param stopwordsFile File to load stop words from
    * @param enablePositionIncrements See {@link
-   * StopFilter#setEnablePositionIncrements} */
+   * StopFilter#setEnablePositionIncrements}
+   * @deprecated Use {@link #StopAnalyzer(Version, File)} instead */
   public StopAnalyzer(File stopwordsFile, boolean enablePositionIncrements) throws IOException {
     stopWords = WordlistLoader.getWordSet(stopwordsFile);
     this.enablePositionIncrements = enablePositionIncrements;
     useDefaultStopPositionIncrement = false;
   }
 
+  /** Builds an analyzer with the stop words from the given file.
+   * @see WordlistLoader#getWordSet(File)
+   * @param matchVersion See <a href="#version">above</a>
+   * @param stopwordsFile File to load stop words from */
+  public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException {
+    stopWords = WordlistLoader.getWordSet(stopwordsFile);
+    this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+    useDefaultStopPositionIncrement = false;
+  }
+
   /** Builds an analyzer with the stop words from the given reader.
    * @see WordlistLoader#getWordSet(Reader)
-   * @deprecated Use {@link #StopAnalyzer(Reader, boolean)} instead
+   * @deprecated Use {@link #StopAnalyzer(Version, Reader)} instead
    */
   public StopAnalyzer(Reader stopwords) throws IOException {
     stopWords = WordlistLoader.getWordSet(stopwords);
@@ -149,13 +189,24 @@
    * @see WordlistLoader#getWordSet(Reader)
    * @param stopwords Reader to load stop words from
    * @param enablePositionIncrements See {@link
-   * StopFilter#setEnablePositionIncrements} */
+   * StopFilter#setEnablePositionIncrements}
+   * @deprecated Use {@link #StopAnalyzer(Version, Reader)} instead */
   public StopAnalyzer(Reader stopwords, boolean enablePositionIncrements) throws IOException {
     stopWords = WordlistLoader.getWordSet(stopwords);
     this.enablePositionIncrements = enablePositionIncrements;
     useDefaultStopPositionIncrement = false;
   }
 
+  /** Builds an analyzer with the stop words from the given reader.
+   * @see WordlistLoader#getWordSet(Reader)
+   * @param matchVersion See <a href="#version">above</a>
+   * @param stopwords Reader to load stop words from */
+  public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
+    stopWords = WordlistLoader.getWordSet(stopwords);
+    this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+    useDefaultStopPositionIncrement = false;
+  }
+
   /** Filters LowerCaseTokenizer with StopFilter. */
   public TokenStream tokenStream(String fieldName, Reader reader) {
     if (useDefaultStopPositionIncrement) {

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.queryParser.QueryParser; // for javadoc
+import org.apache.lucene.util.Version;
 
 /**
  * Removes stop words from a token stream.
@@ -243,6 +244,21 @@
   }
 
   /**
+   * Returns version-dependent default for
+   * enablePositionIncrements.  Analyzers that embed
+   * StopFilter use this method when creating the
+   * StopFilter.  Prior to 2.9, this returns false.  On 2.9
+   * or later, it returns true.
+   */
+  public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) {
+    if (matchVersion.onOrAfter(Version.LUCENE_29)) {
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  /**
    * Set the default position increments behavior of every StopFilter created from now on.
    * <p>
    * Note: behavior of a single StopFilter instance can be modified 

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -35,7 +35,7 @@
  * compatibility when creating StandardAnalyzer:
  * <ul>
  *   <li> As of 2.9, StopFilter preserves position
- *        increments by default
+ *        increments
  *   <li> As of 2.4, Tokens incorrectly identified as acronyms
  *        are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a>
  * </ul>



Mime
View raw message