lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r824116 [1/2] - in /lucene/java/trunk: ./ contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/m...
Date Sun, 11 Oct 2009 17:35:11 GMT
Author: uschindler
Date: Sun Oct 11 17:35:09 2009
New Revision: 824116

URL: http://svn.apache.org/viewvc?rev=824116&view=rev
Log:
LUCENE-1946, LUCENE-1753: Remove deprecated TokenStream API. What a pity, my wonderful backwards layer is gone! :-( Enforce decorator pattern by making the rest of TokenStreams final.

Added:
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java   (props changed)
      - copied unchanged from r821339, lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
Removed:
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizer.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizer.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/SinkTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TeeTokenFilter.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenWrapper.java
Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/build.xml
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
    lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
    lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
    lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
    lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/CharStream.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ParseException.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserConstants.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java
    lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java
    lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
    lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
    lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
    lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
    lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
    lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
    lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Sun Oct 11 17:35:09 2009
@@ -32,17 +32,20 @@
   (Michael Busch)
 
 * LUCENE-1957: Remove Filter.bits(IndexReader) method and make 
-               Filter.getDocIdSet(IndexReader) abstract. (Michael Busch)
+  Filter.getDocIdSet(IndexReader) abstract. (Michael Busch)
 
 * LUCENE-1960: Remove deprecated Field.Store.COMPRESS. (Michael Busch)
 
 * LUCENE-1961: Remove remaining deprecations from document package.
-               (Michael Busch)
+  (Michael Busch)
 
 * LUCENE-1968: Remove deprecated methods in PriorityQueue. (Michael Busch)
 
 * LUCENE-1970: Remove deprecated methods in DocIdSetIterator and make
-               new ones abstract. (Michael Busch)
+  new ones abstract. (Michael Busch)
+
+* LUCENE-1753: Make not yet final TokenStreams final to enforce 
+  decorator pattern (Uwe Schindler)
 
 Bug fixes
 

Modified: lucene/java/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/trunk/build.xml?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/build.xml (original)
+++ lucene/java/trunk/build.xml Sun Oct 11 17:35:09 2009
@@ -577,7 +577,7 @@
     </delete>
   </target>
 
-  <target name="javacc" depends="clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser, javacc-contrib-surround"/>
+  <target name="javacc" depends="clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser, javacc-contrib-surround, javacc-contrib-precedence"/>
 
   <target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
     <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
@@ -605,6 +605,13 @@
     />
   </target>
   
+  <target name="javacc-contrib-precedence" depends="init,javacc-check" if="javacc.present">
+    <ant target="javacc"
+      dir="contrib/misc"
+      antfile="build.xml" 
+    />
+  </target>
+  
   <!-- ================================================================== -->
   <!-- Build the JFlex files into the source tree                         -->
   <!-- ================================================================== -->

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java Sun Oct 11 17:35:09 2009
@@ -164,18 +164,6 @@
     }
   }
   
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-  
   protected static final void addAllLowerCase(Set target, Collection col) {
     Iterator iter=col.iterator();
     

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java Sun Oct 11 17:35:09 2009
@@ -36,7 +36,7 @@
  * 
  * @see <a href="http://fr.wikipedia.org/wiki/%C3%89lision">Elision in Wikipedia</a>
  */
-public class ElisionFilter extends TokenFilter {
+public final class ElisionFilter extends TokenFilter {
   private Set articles = null;
   private TermAttribute termAtt;
   
@@ -109,16 +109,4 @@
       return false;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java Sun Oct 11 17:35:09 2009
@@ -25,21 +25,10 @@
 /**
  * An always exhausted token stream.
  */
-public class EmptyTokenStream extends TokenStream {
+public final class EmptyTokenStream extends TokenStream {
 
   public final boolean incrementToken() throws IOException {
     return false;
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -62,18 +62,6 @@
   public final boolean incrementToken() throws IOException {
     return suffix.incrementToken();
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset() throws IOException {
     suffix.reset();

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -111,18 +111,6 @@
     return true;
   }
   
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-  
   private void setCurrentToken(Token token) {
     if (token == null) return;
     termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java Sun Oct 11 17:35:09 2009
@@ -27,7 +27,7 @@
 /**
  * A {@link TokenStream} containing a single token.
  */
-public class SingleTokenTokenStream extends TokenStream {
+public final class SingleTokenTokenStream extends TokenStream {
 
   private boolean exhausted = false;
   
@@ -42,7 +42,7 @@
     this.singleToken = (Token) token.clone();
     
     tokenAtt = (AttributeImpl) addAttribute(TermAttribute.class);
-    assert (tokenAtt instanceof Token || tokenAtt.getClass().getName().equals("org.apache.lucene.analysis.TokenWrapper"));
+    assert (tokenAtt instanceof Token);
   }
 
   public final boolean incrementToken() throws IOException {
@@ -55,18 +55,6 @@
       return true;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset() throws IOException {
     exhausted = false;

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -31,7 +31,7 @@
  * This {@link TokenFilter} create n-grams from the beginning edge or ending edge of a input token.
  * </p>
  */
-public class EdgeNGramTokenFilter extends TokenFilter {
+public final class EdgeNGramTokenFilter extends TokenFilter {
   public static final Side DEFAULT_SIDE = Side.FRONT;
   public static final int DEFAULT_MAX_GRAM_SIZE = 1;
   public static final int DEFAULT_MIN_GRAM_SIZE = 1;
@@ -149,18 +149,6 @@
       curTermBuffer = null;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset() throws IOException {
     super.reset();

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Sun Oct 11 17:35:09 2009
@@ -33,7 +33,7 @@
  * MaxGram can't be larger than 1024 because of limitation.
  * </p>
  */
-public class EdgeNGramTokenizer extends Tokenizer {
+public final class EdgeNGramTokenizer extends Tokenizer {
   public static final Side DEFAULT_SIDE = Side.FRONT;
   public static final int DEFAULT_MAX_GRAM_SIZE = 1;
   public static final int DEFAULT_MIN_GRAM_SIZE = 1;
@@ -217,18 +217,6 @@
     final int finalOffset = inLen;
     this.offsetAtt.setOffset(finalOffset, finalOffset);
   }    
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset(Reader input) throws IOException {
     super.reset(input);

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -28,7 +28,7 @@
 /**
  * Tokenizes the input into n-grams of the given size(s).
  */
-public class NGramTokenFilter extends TokenFilter {
+public final class NGramTokenFilter extends TokenFilter {
   public static final int DEFAULT_MIN_NGRAM_SIZE = 1;
   public static final int DEFAULT_MAX_NGRAM_SIZE = 2;
 
@@ -97,18 +97,6 @@
       curTermBuffer = null;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset() throws IOException {
     super.reset();

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Sun Oct 11 17:35:09 2009
@@ -29,7 +29,7 @@
 /**
  * Tokenizes the input into n-grams of the given size(s).
  */
-public class NGramTokenizer extends Tokenizer {
+public final class NGramTokenizer extends Tokenizer {
   public static final int DEFAULT_MIN_NGRAM_SIZE = 1;
   public static final int DEFAULT_MAX_NGRAM_SIZE = 2;
 
@@ -134,18 +134,6 @@
     this.offsetAtt.setOffset(finalOffset, finalOffset);
   }    
   
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-  
   public void reset(Reader input) throws IOException {
     super.reset(input);
     reset();

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -57,16 +57,4 @@
       return false;
     }
   }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -55,16 +55,4 @@
     return false;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }
\ No newline at end of file

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -55,16 +55,4 @@
       return false;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }
\ No newline at end of file

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Sun Oct 11 17:35:09 2009
@@ -28,7 +28,7 @@
  * except the first return token which retains its original positionIncrement value.
  * The default positionIncrement value is zero.
  */
-public class PositionFilter extends TokenFilter {
+public final class PositionFilter extends TokenFilter {
 
   /** Position increment to assign to all but the first token - default = 0 */
   private int positionIncrement = 0;
@@ -75,18 +75,6 @@
     }
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-
   public void reset() throws IOException {
     super.reset();
     firstTokenPositioned = false;

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Sun Oct 11 17:35:09 2009
@@ -41,7 +41,7 @@
  * <p>This filter handles position increments > 1 by inserting filler tokens
  * (tokens with termtext "_"). It does not handle a position increment of 0.
  */
-public class ShingleFilter extends TokenFilter {
+public final class ShingleFilter extends TokenFilter {
 
   private LinkedList shingleBuf = new LinkedList();
   private StringBuilder[] shingles;
@@ -327,18 +327,6 @@
     
     return true;
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 
   public void reset() throws IOException {
     super.reset();

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java Sun Oct 11 17:35:09 2009
@@ -114,7 +114,7 @@
  * <b>NOTE:</b> This filter might not behave correctly if used with custom Attributes, i.e. Attributes other than
  * the ones located in org.apache.lucene.analysis.tokenattributes.
  */
-public class ShingleMatrixFilter extends TokenStream {
+public final class ShingleMatrixFilter extends TokenStream {
 
   public static Character defaultSpacerCharacter = new Character('_');
   public static TokenSettingsCodec defaultSettingsCodec = new OneDimensionalNonWeightedTokenSettingsCodec();
@@ -393,16 +393,15 @@
     return token;
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
+  private Token getNextToken(Token token) throws IOException {
+    if (!this.incrementToken()) return null;
+    token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
+    token.setPositionIncrement(posIncrAtt.getPositionIncrement());
+    token.setFlags(flagsAtt.getFlags());
+    token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
+    token.setType(typeAtt.type());
+    token.setPayload(payloadAtt.getPayload());
+    return token;
   }
 
   private static final Token request_next_token = new Token();
@@ -429,7 +428,7 @@
         if (ignoringSinglePrefixOrSuffixShingle
             && currentShingleLength == 1
             && ((currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isFirst() || (currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isLast())) {
-          return next(reusableToken);
+          return getNextToken(reusableToken);
         }
 
         int termLength = 0;

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Sun Oct 11 17:35:09 2009
@@ -32,7 +32,7 @@
  * Token that is Thai into separate Token(s) for each Thai word.
  * @version 0.2
  */
-public class ThaiWordFilter extends TokenFilter {
+public final class ThaiWordFilter extends TokenFilter {
   
   private BreakIterator breaker = null;
   
@@ -82,18 +82,6 @@
     return false;
   }
   
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-  
   public void reset() throws IOException {
     super.reset();
     thaiState = null;

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -20,25 +20,29 @@
 import java.io.IOException;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 public class TestSingleTokenTokenFilter extends LuceneTestCase {
 
   public void test() throws IOException {
-    final Token reusableToken = new Token();
-    
     Token token = new Token();
     SingleTokenTokenStream ts = new SingleTokenTokenStream(token);
+    AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(TermAttribute.class);
+    assertTrue(tokenAtt instanceof Token);
     ts.reset();
 
-    assertEquals(token, ts.next(reusableToken));
-    assertNull(ts.next(reusableToken));
+    assertTrue(ts.incrementToken());
+    assertEquals(token, tokenAtt);
+    assertFalse(ts.incrementToken());
     
     token = new Token("hallo", 10, 20, "someType");
     ts.setToken(token);
     ts.reset();
 
-    assertEquals(token, ts.next(reusableToken));
-    assertNull(ts.next(reusableToken));
+    assertTrue(ts.incrementToken());
+    assertEquals(token, tokenAtt);
+    assertFalse(ts.incrementToken());
   }
 }

Modified: lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/java/trunk/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Sun Oct 11 17:35:09 2009
@@ -21,7 +21,6 @@
 import java.util.List;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.benchmark.byTask.PerfRunData;
 import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
@@ -59,8 +58,6 @@
     super.tearDown();
   }
 
-  Token token = new Token();
-
   public int doLogic() throws Exception {
     List fields = doc.getFields();
     final int numField = fields.size();
@@ -96,7 +93,7 @@
       // reset the TokenStream to the first token
       stream.reset();
 
-      while(stream.next(token) != null)
+      while(stream.incrementToken())
         tokenCount++;
     }
     totalTokenCount += tokenCount;

Modified: lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/java/trunk/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Sun Oct 11 17:35:09 2009
@@ -45,6 +45,7 @@
 import org.apache.lucene.search.DefaultSimilarity;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.AttributeImpl;
 
 /**
  * This class, similar to {@link org.apache.lucene.index.IndexWriter}, has no locking mechanism.
@@ -524,14 +525,27 @@
           // reset the TokenStream to the first token          
           tokenStream.reset();
 
-          final Token reusableToken = new Token();
-          for (Token nextToken = tokenStream.next(reusableToken); nextToken != null; nextToken = tokenStream.next(reusableToken)) {
-            tokens.add((Token) nextToken.clone()); // the vector will be built on commit.
+          while (tokenStream.incrementToken()) {
+            // TODO: this is a simple workaround to still work with tokens, not very effective, but as far as I know, this writer should get removed soon:
+            final Token token = new Token();
+            for (Iterator<AttributeImpl> atts = tokenStream.getAttributeImplsIterator(); atts.hasNext();) {
+              final AttributeImpl att = atts.next();
+              try {
+                att.copyTo(token);
+              } catch (Exception e) {
+                // ignore unsupported attributes,
+                // this may fail to copy some attributes, if a special combined AttributeImpl is used, that
+                // implements basic attributes supported by Token and also other customized ones in one class.
+              }
+            }
+            tokens.add(token); // the vector will be built on commit.
             fieldSetting.fieldLength++;
             if (fieldSetting.fieldLength > maxFieldLength) {
               break;
             }
           }
+          tokenStream.end();
+          tokenStream.close();
         } else {
           // untokenized
           String fieldVal = field.stringValue();

Modified: lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (original)
+++ lucene/java/trunk/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java Sun Oct 11 17:35:09 2009
@@ -39,6 +39,7 @@
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.*;
 
 /**
 Verifies that Lucene PatternAnalyzer and normal Lucene Analyzers have the same behaviour,
@@ -141,16 +142,30 @@
               boolean toLowerCase = toLower != 0;
                 
               for (int run=0; run < runs; run++) {
-                List tokens1 = null; List tokens2 = null;
+                TokenStream tokens1 = null; TokenStream tokens2 = null;
                 try {
-                  if (usePattern) tokens1 = getTokens(patternTokenStream(text, lettersOnly, toLowerCase, stopWords));
-                  if (useLucene) tokens2 = getTokens(luceneTokenStream(text, lettersOnly, toLowerCase, stopWords));          
-                  if (usePattern && useLucene) assertEquals(tokens1, tokens2);
+                  if (usePattern) tokens1 = patternTokenStream(text, lettersOnly, toLowerCase, stopWords);
+                  if (useLucene) tokens2 = luceneTokenStream(text, lettersOnly, toLowerCase, stopWords);          
+                  if (usePattern && useLucene) {
+                    final TermAttribute termAtt1 = tokens1.addAttribute(TermAttribute.class),
+                      termAtt2 = tokens2.addAttribute(TermAttribute.class);
+                    final OffsetAttribute offsetAtt1 = tokens1.addAttribute(OffsetAttribute.class),
+                      offsetAtt2 = tokens2.addAttribute(OffsetAttribute.class);
+                    final PositionIncrementAttribute posincrAtt1 = tokens1.addAttribute(PositionIncrementAttribute.class),
+                      posincrAtt2 = tokens2.addAttribute(PositionIncrementAttribute.class);
+                    while (tokens1.incrementToken()) {
+                      assertTrue(tokens2.incrementToken());
+                      assertEquals(termAtt1, termAtt2);
+                      assertEquals(offsetAtt1, offsetAtt2);
+                      assertEquals(posincrAtt1, posincrAtt2);
+                    }
+                    assertFalse(tokens2.incrementToken());
+                    tokens1.end(); tokens1.close();
+                    tokens2.end(); tokens2.close();
+                  }
                 } catch (Throwable t) {
                   if (t instanceof OutOfMemoryError) t.printStackTrace();
                   System.out.println("fatal error at file=" + file + ", letters="+ lettersOnly + ", toLowerCase=" + toLowerCase + ", stopwords=" + (stopWords != null ? "english" : "none"));
-                  System.out.println("\n\ntokens1=" + toString(tokens1));
-                  System.out.println("\n\ntokens2=" + toString(tokens2));
                   throw t;
                 }
               }
@@ -194,50 +209,6 @@
     return stream;            
   }
   
-  private List getTokens(TokenStream stream) throws IOException {
-    ArrayList tokens = new ArrayList();
-    final Token reusableToken = new Token();
-    for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
-      tokens.add(nextToken.clone());
-    }
-    return tokens;
-  }
-  
-  private void assertEquals(List tokens1, List tokens2) {
-    int size = Math.min(tokens1.size(), tokens2.size());
-    int i=0;
-    try {
-      for (; i < size; i++) {
-        Token t1 = (Token) tokens1.get(i);
-        Token t2 = (Token) tokens2.get(i);
-        if (!(t1.term().equals(t2.term()))) throw new IllegalStateException("termText");
-        if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset");
-        if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset");
-        if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type");
-      }
-      if (tokens1.size() != tokens2.size())   throw new IllegalStateException("size1=" + tokens1.size() + ", size2=" + tokens2.size());
-    }
-
-    catch (IllegalStateException e) {
-      if (size > 0) {
-        System.out.println("i=" + i + ", size=" + size);
-        System.out.println("t1[size]='" + ((Token) tokens1.get(size-1)).term() + "'");
-        System.out.println("t2[size]='" + ((Token) tokens2.get(size-1)).term() + "'");
-      }
-      throw e;
-    }
-  }
-  
-  private String toString(List tokens) {
-    if (tokens == null) return "null";
-    String str = "[";
-    for (int i=0; i < tokens.size(); i++) {
-      Token t1 = (Token) tokens.get(i);
-      str = str + "'" + t1.term() + "', ";
-    }
-    return str + "]";
-  }
-  
   // trick to detect default platform charset
   private static final Charset DEFAULT_PLATFORM_CHARSET = 
     Charset.forName(new InputStreamReader(new ByteArrayInputStream(new byte[0])).getEncoding());  

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/CharStream.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/CharStream.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/CharStream.java Sun Oct 11 17:35:09 2009
@@ -1,4 +1,5 @@
-/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. CharStream.java Version 4.1 */
+/* JavaCCOptions:STATIC=false */
 package org.apache.lucene.queryParser.precedence;
 
 /**
@@ -27,14 +28,14 @@
 
   /**
    * Returns the column position of the character last read.
-   * @deprecated 
+   * @deprecated
    * @see #getEndColumn
    */
   int getColumn();
 
   /**
    * Returns the line number of the character last read.
-   * @deprecated 
+   * @deprecated
    * @see #getEndLine
    */
   int getLine();
@@ -79,7 +80,7 @@
   char BeginToken() throws java.io.IOException;
 
   /**
-   * Returns a string made up of characters from the marked token beginning 
+   * Returns a string made up of characters from the marked token beginning
    * to the current buffer position. Implementations have the choice of returning
    * anything that they want to. For example, for efficiency, one might decide
    * to just return null, which is a valid implementation.
@@ -108,3 +109,4 @@
   void Done();
 
 }
+/* JavaCC - OriginalChecksum=8cc617b193267dc876ef9699367c8186 (do not edit this line) */

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ParseException.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ParseException.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/ParseException.java Sun Oct 11 17:35:09 2009
@@ -1,4 +1,5 @@
-/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 4.1 */
+/* JavaCCOptions:KEEP_LINE_COL=null */
 package org.apache.lucene.queryParser.precedence;
 
 /**
@@ -51,6 +52,7 @@
     specialConstructor = false;
   }
 
+  /** Constructor with message. */
   public ParseException(String message) {
     super(message);
     specialConstructor = false;
@@ -98,19 +100,19 @@
     if (!specialConstructor) {
       return super.getMessage();
     }
-    String expected = "";
+    StringBuffer expected = new StringBuffer();
     int maxSize = 0;
     for (int i = 0; i < expectedTokenSequences.length; i++) {
       if (maxSize < expectedTokenSequences[i].length) {
         maxSize = expectedTokenSequences[i].length;
       }
       for (int j = 0; j < expectedTokenSequences[i].length; j++) {
-        expected += tokenImage[expectedTokenSequences[i][j]] + " ";
+        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
       }
       if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
-        expected += "...";
+        expected.append("...");
       }
-      expected += eol + "    ";
+      expected.append(eol).append("    ");
     }
     String retval = "Encountered \"";
     Token tok = currentToken.next;
@@ -120,8 +122,11 @@
         retval += tokenImage[0];
         break;
       }
+      retval += " " + tokenImage[tok.kind];
+      retval += " \"";
       retval += add_escapes(tok.image);
-      tok = tok.next; 
+      retval += " \"";
+      tok = tok.next;
     }
     retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
     retval += "." + eol;
@@ -130,7 +135,7 @@
     } else {
       retval += "Was expecting one of:" + eol + "    ";
     }
-    retval += expected;
+    retval += expected.toString();
     return retval;
   }
 
@@ -138,7 +143,7 @@
    * The end of line string for this machine.
    */
   protected String eol = System.getProperty("line.separator", "\n");
- 
+
   /**
    * Used to convert raw characters to their escaped version
    * when these raw version cannot be used as part of an ASCII
@@ -190,3 +195,4 @@
    }
 
 }
+/* JavaCC - OriginalChecksum=15fbbe38a36c8ac9e2740d030624c321 (do not edit this line) */

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Sun Oct 11 17:35:09 2009
@@ -12,6 +12,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.*;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
@@ -25,6 +26,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.Parameter;
+import org.apache.lucene.util.AttributeSource;
 
 /**
  * Experimental query parser variant designed to handle operator precedence
@@ -70,6 +72,9 @@
  * documentation</a>.
  * </p>
  *
+ * @author Brian Goetz
+ * @author Peter Halacsy
+ * @author Tatu Saloranta
  */
 
 public class PrecedenceQueryParser implements PrecedenceQueryParserConstants {
@@ -205,7 +210,7 @@
    * In default mode (<code>OR_OPERATOR</code>) terms without any modifiers
    * are considered optional: for example <code>capital of Hungary</code> is equal to
    * <code>capital OR of OR Hungary</code>.<br/>
-   * In <code>AND_OPERATOR</code> mode terms are considered to be in conjuction: the
+   * In <code>AND_OPERATOR</code> mode terms are considered to be in conjunction: the
    * above mentioned query is parsed as <code>capital AND of AND Hungary</code>
    */
   public void setDefaultOperator(Operator op) {
@@ -314,48 +319,40 @@
     // PhraseQuery, or nothing based on the term count
 
     TokenStream source = analyzer.tokenStream(field, new StringReader(queryText));
-    List list = new ArrayList();
-    final org.apache.lucene.analysis.Token reusableToken = new org.apache.lucene.analysis.Token();
-    org.apache.lucene.analysis.Token nextToken;
+    List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
     int positionCount = 0;
     boolean severalTokensAtSamePosition = false;
+    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
 
-    while (true) {
-      try {
-        nextToken = source.next(reusableToken);
-      }
-      catch (IOException e) {
-        nextToken = null;
-      }
-      if (nextToken == null)
-        break;
-      list.add(nextToken.clone());
-      if (nextToken.getPositionIncrement() == 1)
-        positionCount++;
-      else
-        severalTokensAtSamePosition = true;
-    }
     try {
+      while (source.incrementToken()) {
+        list.add(source.captureState());
+        if (posincrAtt.getPositionIncrement() == 1)
+          positionCount++;
+        else
+          severalTokensAtSamePosition = true;
+      }
+      source.end();
       source.close();
-    }
-    catch (IOException e) {
-      // ignore
+    } catch (IOException e) {
+      // ignore, should never happen for StringReaders
     }
 
     if (list.size() == 0)
       return null;
     else if (list.size() == 1) {
-      nextToken = (org.apache.lucene.analysis.Token) list.get(0);
-      return new TermQuery(new Term(field, nextToken.term()));
+      source.restoreState(list.get(0));
+      return new TermQuery(new Term(field, termAtt.term()));
     } else {
       if (severalTokensAtSamePosition) {
         if (positionCount == 1) {
           // no phrase query:
           BooleanQuery q = new BooleanQuery();
           for (int i = 0; i < list.size(); i++) {
-            nextToken = (org.apache.lucene.analysis.Token) list.get(i);
+            source.restoreState(list.get(i));
             TermQuery currentQuery = new TermQuery(
-                new Term(field, nextToken.term()));
+                new Term(field, termAtt.term()));
             q.add(currentQuery, BooleanClause.Occur.SHOULD);
           }
           return q;
@@ -363,14 +360,14 @@
         else {
           // phrase query:
           MultiPhraseQuery mpq = new MultiPhraseQuery();
-          List multiTerms = new ArrayList();
+          List<Term> multiTerms = new ArrayList<Term>();
           for (int i = 0; i < list.size(); i++) {
-            nextToken = (org.apache.lucene.analysis.Token) list.get(i);
-            if (nextToken.getPositionIncrement() == 1 && multiTerms.size() > 0) {
+            source.restoreState(list.get(i));
+            if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) {
               mpq.add((Term[])multiTerms.toArray(new Term[0]));
               multiTerms.clear();
             }
-            multiTerms.add(new Term(field, nextToken.term()));
+            multiTerms.add(new Term(field, termAtt.term()));
           }
           mpq.add((Term[])multiTerms.toArray(new Term[0]));
           return mpq;
@@ -380,8 +377,8 @@
         PhraseQuery q = new PhraseQuery();
         q.setSlop(phraseSlop);
         for (int i = 0; i < list.size(); i++) {
-          q.add(new Term(field, ((org.apache.lucene.analysis.Token)
-              list.get(i)).term()));
+          source.restoreState(list.get(i));
+          q.add(new Term(field, termAtt.term()));
         }
         return q;
       }
@@ -1093,39 +1090,42 @@
     throw new Error("Missing return statement in function");
   }
 
-  final private boolean jj_2_1(int xla) {
+  private boolean jj_2_1(int xla) {
     jj_la = xla; jj_lastpos = jj_scanpos = token;
     try { return !jj_3_1(); }
     catch(LookaheadSuccess ls) { return true; }
     finally { jj_save(0, xla); }
   }
 
-  final private boolean jj_3_1() {
+  private boolean jj_3_1() {
     if (jj_scan_token(TERM)) return true;
     if (jj_scan_token(COLON)) return true;
     return false;
   }
 
+  /** Generated Token Manager. */
   public PrecedenceQueryParserTokenManager token_source;
-  public Token token, jj_nt;
+  /** Current token. */
+  public Token token;
+  /** Next token. */
+  public Token jj_nt;
   private int jj_ntk;
   private Token jj_scanpos, jj_lastpos;
   private int jj_la;
-  public boolean lookingAhead = false;
-  private boolean jj_semLA;
   private int jj_gen;
   final private int[] jj_la1 = new int[24];
   static private int[] jj_la1_0;
   static {
-      jj_la1_0();
+      jj_la1_init_0();
    }
-   private static void jj_la1_0() {
+   private static void jj_la1_init_0() {
       jj_la1_0 = new int[] {0x180,0x180,0xe00,0xe00,0xfb1f00,0x100,0x80,0x8000,0xfb1000,0x9a0000,0x40000,0x40000,0x8000,0xc000000,0x1000000,0xc000000,0x8000,0xc0000000,0x10000000,0xc0000000,0x8000,0x40000,0x8000,0xfb0000,};
    }
   final private JJCalls[] jj_2_rtns = new JJCalls[1];
   private boolean jj_rescan = false;
   private int jj_gc = 0;
 
+  /** Constructor with user supplied CharStream. */
   public PrecedenceQueryParser(CharStream stream) {
     token_source = new PrecedenceQueryParserTokenManager(stream);
     token = new Token();
@@ -1135,6 +1135,7 @@
     for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
   }
 
+  /** Reinitialise. */
   public void ReInit(CharStream stream) {
     token_source.ReInit(stream);
     token = new Token();
@@ -1144,6 +1145,7 @@
     for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
   }
 
+  /** Constructor with generated Token Manager. */
   public PrecedenceQueryParser(PrecedenceQueryParserTokenManager tm) {
     token_source = tm;
     token = new Token();
@@ -1153,6 +1155,7 @@
     for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
   }
 
+  /** Reinitialise. */
   public void ReInit(PrecedenceQueryParserTokenManager tm) {
     token_source = tm;
     token = new Token();
@@ -1162,7 +1165,7 @@
     for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
   }
 
-  final private Token jj_consume_token(int kind) throws ParseException {
+  private Token jj_consume_token(int kind) throws ParseException {
     Token oldToken;
     if ((oldToken = token).next != null) token = token.next;
     else token = token.next = token_source.getNextToken();
@@ -1188,7 +1191,7 @@
 
   static private final class LookaheadSuccess extends java.lang.Error { }
   final private LookaheadSuccess jj_ls = new LookaheadSuccess();
-  final private boolean jj_scan_token(int kind) {
+  private boolean jj_scan_token(int kind) {
     if (jj_scanpos == jj_lastpos) {
       jj_la--;
       if (jj_scanpos.next == null) {
@@ -1209,6 +1212,8 @@
     return false;
   }
 
+
+/** Get the next Token. */
   final public Token getNextToken() {
     if (token.next != null) token = token.next;
     else token = token.next = token_source.getNextToken();
@@ -1217,8 +1222,9 @@
     return token;
   }
 
+/** Get the specific Token. */
   final public Token getToken(int index) {
-    Token t = lookingAhead ? jj_scanpos : token;
+    Token t = token;
     for (int i = 0; i < index; i++) {
       if (t.next != null) t = t.next;
       else t = t.next = token_source.getNextToken();
@@ -1226,14 +1232,14 @@
     return t;
   }
 
-  final private int jj_ntk() {
+  private int jj_ntk() {
     if ((jj_nt=token.next) == null)
       return (jj_ntk = (token.next=token_source.getNextToken()).kind);
     else
       return (jj_ntk = jj_nt.kind);
   }
 
-  private java.util.Vector jj_expentries = new java.util.Vector();
+  private java.util.List jj_expentries = new java.util.ArrayList();
   private int[] jj_expentry;
   private int jj_kind = -1;
   private int[] jj_lasttokens = new int[100];
@@ -1248,31 +1254,26 @@
       for (int i = 0; i < jj_endpos; i++) {
         jj_expentry[i] = jj_lasttokens[i];
       }
-      boolean exists = false;
-      for (java.util.Enumeration e = jj_expentries.elements(); e.hasMoreElements();) {
-        int[] oldentry = (int[])(e.nextElement());
+      jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
+        int[] oldentry = (int[])(it.next());
         if (oldentry.length == jj_expentry.length) {
-          exists = true;
           for (int i = 0; i < jj_expentry.length; i++) {
             if (oldentry[i] != jj_expentry[i]) {
-              exists = false;
-              break;
+              continue jj_entries_loop;
             }
           }
-          if (exists) break;
+          jj_expentries.add(jj_expentry);
+          break jj_entries_loop;
         }
       }
-      if (!exists) jj_expentries.addElement(jj_expentry);
       if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
     }
   }
 
+  /** Generate ParseException. */
   public ParseException generateParseException() {
-    jj_expentries.removeAllElements();
+    jj_expentries.clear();
     boolean[] la1tokens = new boolean[32];
-    for (int i = 0; i < 32; i++) {
-      la1tokens[i] = false;
-    }
     if (jj_kind >= 0) {
       la1tokens[jj_kind] = true;
       jj_kind = -1;
@@ -1290,7 +1291,7 @@
       if (la1tokens[i]) {
         jj_expentry = new int[1];
         jj_expentry[0] = i;
-        jj_expentries.addElement(jj_expentry);
+        jj_expentries.add(jj_expentry);
       }
     }
     jj_endpos = 0;
@@ -1298,20 +1299,23 @@
     jj_add_error_token(0, 0);
     int[][] exptokseq = new int[jj_expentries.size()][];
     for (int i = 0; i < jj_expentries.size(); i++) {
-      exptokseq[i] = (int[])jj_expentries.elementAt(i);
+      exptokseq[i] = (int[])jj_expentries.get(i);
     }
     return new ParseException(token, exptokseq, tokenImage);
   }
 
+  /** Enable tracing. */
   final public void enable_tracing() {
   }
 
+  /** Disable tracing. */
   final public void disable_tracing() {
   }
 
-  final private void jj_rescan_token() {
+  private void jj_rescan_token() {
     jj_rescan = true;
     for (int i = 0; i < 1; i++) {
+    try {
       JJCalls p = jj_2_rtns[i];
       do {
         if (p.gen > jj_gen) {
@@ -1322,11 +1326,12 @@
         }
         p = p.next;
       } while (p != null);
+      } catch(LookaheadSuccess ls) { }
     }
     jj_rescan = false;
   }
 
-  final private void jj_save(int index, int xla) {
+  private void jj_save(int index, int xla) {
     JJCalls p = jj_2_rtns[index];
     while (p.gen > jj_gen) {
       if (p.next == null) { p = p.next = new JJCalls(); break; }

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj Sun Oct 11 17:35:09 2009
@@ -36,6 +36,7 @@
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.*;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
@@ -49,6 +50,7 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.Parameter;
+import org.apache.lucene.util.AttributeSource;
 
 /**
  * Experimental query parser variant designed to handle operator precedence
@@ -341,48 +343,40 @@
     // PhraseQuery, or nothing based on the term count
 
     TokenStream source = analyzer.tokenStream(field, new StringReader(queryText));
-    List list = new ArrayList();
-    final org.apache.lucene.analysis.Token reusableToken = new org.apache.lucene.analysis.Token();
-    org.apache.lucene.analysis.Token nextToken;
+    List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
     int positionCount = 0;
     boolean severalTokensAtSamePosition = false;
+    TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+    PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
 
-    while (true) {
-      try {
-        nextToken = source.next(reusableToken);
-      }
-      catch (IOException e) {
-        nextToken = null;
-      }
-      if (nextToken == null)
-        break;
-      list.add(nextToken.clone());
-      if (nextToken.getPositionIncrement() == 1)
-        positionCount++;
-      else
-        severalTokensAtSamePosition = true;
-    }
     try {
+      while (source.incrementToken()) {
+        list.add(source.captureState());
+        if (posincrAtt.getPositionIncrement() == 1)
+          positionCount++;
+        else
+          severalTokensAtSamePosition = true;
+      }
+      source.end();
       source.close();
-    }
-    catch (IOException e) {
-      // ignore
+    } catch (IOException e) {
+      // ignore, should never happen for StringReaders
     }
 
     if (list.size() == 0)
       return null;
     else if (list.size() == 1) {
-      nextToken = (org.apache.lucene.analysis.Token) list.get(0);
-      return new TermQuery(new Term(field, nextToken.term()));
+      source.restoreState(list.get(0));
+      return new TermQuery(new Term(field, termAtt.term()));
     } else {
       if (severalTokensAtSamePosition) {
         if (positionCount == 1) {
           // no phrase query:
           BooleanQuery q = new BooleanQuery();
           for (int i = 0; i < list.size(); i++) {
-            nextToken = (org.apache.lucene.analysis.Token) list.get(i);
+            source.restoreState(list.get(i));
             TermQuery currentQuery = new TermQuery(
-                new Term(field, nextToken.term()));
+                new Term(field, termAtt.term()));
             q.add(currentQuery, BooleanClause.Occur.SHOULD);
           }
           return q;
@@ -390,14 +384,14 @@
         else {
           // phrase query:
           MultiPhraseQuery mpq = new MultiPhraseQuery();
-          List multiTerms = new ArrayList();
+          List<Term> multiTerms = new ArrayList<Term>();
           for (int i = 0; i < list.size(); i++) {
-            nextToken = (org.apache.lucene.analysis.Token) list.get(i);
-            if (nextToken.getPositionIncrement() == 1 && multiTerms.size() > 0) {
+            source.restoreState(list.get(i));
+            if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) {
               mpq.add((Term[])multiTerms.toArray(new Term[0]));
               multiTerms.clear();
             }
-            multiTerms.add(new Term(field, nextToken.term()));
+            multiTerms.add(new Term(field, termAtt.term()));
           }
           mpq.add((Term[])multiTerms.toArray(new Term[0]));
           return mpq;
@@ -407,8 +401,8 @@
         PhraseQuery q = new PhraseQuery();
         q.setSlop(phraseSlop);
         for (int i = 0; i < list.size(); i++) {
-          q.add(new Term(field, ((org.apache.lucene.analysis.Token)
-              list.get(i)).term()));
+          source.restoreState(list.get(i));
+          q.add(new Term(field, termAtt.term()));
         }
         return q;
       }
@@ -700,7 +694,7 @@
 }
 
 <DEFAULT, RangeIn, RangeEx> SKIP : {
-  <<_WHITESPACE>>
+  < <_WHITESPACE>>
 }
 
 // OG: to support prefix queries:

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserConstants.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserConstants.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserConstants.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserConstants.java Sun Oct 11 17:35:09 2009
@@ -1,45 +1,86 @@
 /* Generated By:JavaCC: Do not edit this line. PrecedenceQueryParserConstants.java */
 package org.apache.lucene.queryParser.precedence;
 
+
+/**
+ * Token literal values and constants.
+ * Generated by org.javacc.parser.OtherFilesGen#start()
+ */
 public interface PrecedenceQueryParserConstants {
 
+  /** End of File. */
   int EOF = 0;
+  /** RegularExpression Id. */
   int _NUM_CHAR = 1;
+  /** RegularExpression Id. */
   int _ESCAPED_CHAR = 2;
+  /** RegularExpression Id. */
   int _TERM_START_CHAR = 3;
+  /** RegularExpression Id. */
   int _TERM_CHAR = 4;
+  /** RegularExpression Id. */
   int _WHITESPACE = 5;
+  /** RegularExpression Id. */
   int AND = 7;
+  /** RegularExpression Id. */
   int OR = 8;
+  /** RegularExpression Id. */
   int NOT = 9;
+  /** RegularExpression Id. */
   int PLUS = 10;
+  /** RegularExpression Id. */
   int MINUS = 11;
+  /** RegularExpression Id. */
   int LPAREN = 12;
+  /** RegularExpression Id. */
   int RPAREN = 13;
+  /** RegularExpression Id. */
   int COLON = 14;
+  /** RegularExpression Id. */
   int CARAT = 15;
+  /** RegularExpression Id. */
   int QUOTED = 16;
+  /** RegularExpression Id. */
   int TERM = 17;
+  /** RegularExpression Id. */
   int FUZZY_SLOP = 18;
+  /** RegularExpression Id. */
   int PREFIXTERM = 19;
+  /** RegularExpression Id. */
   int WILDTERM = 20;
+  /** RegularExpression Id. */
   int RANGEIN_START = 21;
+  /** RegularExpression Id. */
   int RANGEEX_START = 22;
+  /** RegularExpression Id. */
   int NUMBER = 23;
+  /** RegularExpression Id. */
   int RANGEIN_TO = 24;
+  /** RegularExpression Id. */
   int RANGEIN_END = 25;
+  /** RegularExpression Id. */
   int RANGEIN_QUOTED = 26;
+  /** RegularExpression Id. */
   int RANGEIN_GOOP = 27;
+  /** RegularExpression Id. */
   int RANGEEX_TO = 28;
+  /** RegularExpression Id. */
   int RANGEEX_END = 29;
+  /** RegularExpression Id. */
   int RANGEEX_QUOTED = 30;
+  /** RegularExpression Id. */
   int RANGEEX_GOOP = 31;
 
+  /** Lexical state. */
   int Boost = 0;
+  /** Lexical state. */
   int RangeEx = 1;
+  /** Lexical state. */
   int RangeIn = 2;
+  /** Lexical state. */
   int DEFAULT = 3;
 
+  /** Literal token values. */
   String[] tokenImage = {
     "<EOF>",
     "<_NUM_CHAR>",

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java Sun Oct 11 17:35:09 2009
@@ -10,6 +10,7 @@
 import java.util.Vector;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.*;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
@@ -23,10 +24,15 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.Parameter;
+import org.apache.lucene.util.AttributeSource;
 
+/** Token Manager. */
 public class PrecedenceQueryParserTokenManager implements PrecedenceQueryParserConstants
 {
+
+  /** Debug output. */
   public  java.io.PrintStream debugStream = System.out;
+  /** Set debug output. */
   public  void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
 private final int jjStopStringLiteralDfa_3(int pos, long active0)
 {
@@ -40,21 +46,13 @@
 {
    return jjMoveNfa_3(jjStopStringLiteralDfa_3(pos, active0), pos + 1);
 }
-private final int jjStopAtPos(int pos, int kind)
+private int jjStopAtPos(int pos, int kind)
 {
    jjmatchedKind = kind;
    jjmatchedPos = pos;
    return pos + 1;
 }
-private final int jjStartNfaWithStates_3(int pos, int kind, int state)
-{
-   jjmatchedKind = kind;
-   jjmatchedPos = pos;
-   try { curChar = input_stream.readChar(); }
-   catch(java.io.IOException e) { return pos + 1; }
-   return jjMoveNfa_3(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_3()
+private int jjMoveStringLiteralDfa0_3()
 {
    switch(curChar)
    {
@@ -78,50 +76,19 @@
          return jjMoveNfa_3(0, 0);
    }
 }
-private final void jjCheckNAdd(int state)
-{
-   if (jjrounds[state] != jjround)
-   {
-      jjstateSet[jjnewStateCnt++] = state;
-      jjrounds[state] = jjround;
-   }
-}
-private final void jjAddStates(int start, int end)
-{
-   do {
-      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
-   } while (start++ != end);
-}
-private final void jjCheckNAddTwoStates(int state1, int state2)
-{
-   jjCheckNAdd(state1);
-   jjCheckNAdd(state2);
-}
-private final void jjCheckNAddStates(int start, int end)
-{
-   do {
-      jjCheckNAdd(jjnextStates[start]);
-   } while (start++ != end);
-}
-private final void jjCheckNAddStates(int start)
-{
-   jjCheckNAdd(jjnextStates[start]);
-   jjCheckNAdd(jjnextStates[start + 1]);
-}
 static final long[] jjbitVec0 = {
    0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
 };
 static final long[] jjbitVec2 = {
    0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
 };
-private final int jjMoveNfa_3(int startState, int curPos)
+private int jjMoveNfa_3(int startState, int curPos)
 {
-   int[] nextStates;
    int startsAt = 0;
    jjnewStateCnt = 33;
    int i = 1;
    jjstateSet[0] = startState;
-   int j, kind = 0x7fffffff;
+   int kind = 0x7fffffff;
    for (;;)
    {
       if (++jjround == 0x7fffffff)
@@ -129,7 +96,7 @@
       if (curChar < 64)
       {
          long l = 1L << curChar;
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -251,7 +218,7 @@
       else if (curChar < 128)
       {
          long l = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -399,7 +366,7 @@
          long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -468,15 +435,7 @@
 {
    return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
 }
-private final int jjStartNfaWithStates_1(int pos, int kind, int state)
-{
-   jjmatchedKind = kind;
-   jjmatchedPos = pos;
-   try { curChar = input_stream.readChar(); }
-   catch(java.io.IOException e) { return pos + 1; }
-   return jjMoveNfa_1(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_1()
+private int jjMoveStringLiteralDfa0_1()
 {
    switch(curChar)
    {
@@ -488,7 +447,7 @@
          return jjMoveNfa_1(0, 0);
    }
 }
-private final int jjMoveStringLiteralDfa1_1(long active0)
+private int jjMoveStringLiteralDfa1_1(long active0)
 {
    try { curChar = input_stream.readChar(); }
    catch(java.io.IOException e) {
@@ -506,14 +465,21 @@
    }
    return jjStartNfa_1(0, active0);
 }
-private final int jjMoveNfa_1(int startState, int curPos)
+private int jjStartNfaWithStates_1(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_1(state, pos + 1);
+}
+private int jjMoveNfa_1(int startState, int curPos)
 {
-   int[] nextStates;
    int startsAt = 0;
    jjnewStateCnt = 5;
    int i = 1;
    jjstateSet[0] = startState;
-   int j, kind = 0x7fffffff;
+   int kind = 0x7fffffff;
    for (;;)
    {
       if (++jjround == 0x7fffffff)
@@ -521,7 +487,7 @@
       if (curChar < 64)
       {
          long l = 1L << curChar;
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -566,7 +532,7 @@
       else if (curChar < 128)
       {
          long l = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -592,7 +558,7 @@
          long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -625,18 +591,17 @@
       catch(java.io.IOException e) { return curPos; }
    }
 }
-private final int jjMoveStringLiteralDfa0_0()
+private int jjMoveStringLiteralDfa0_0()
 {
    return jjMoveNfa_0(0, 0);
 }
-private final int jjMoveNfa_0(int startState, int curPos)
+private int jjMoveNfa_0(int startState, int curPos)
 {
-   int[] nextStates;
    int startsAt = 0;
    jjnewStateCnt = 3;
    int i = 1;
    jjstateSet[0] = startState;
-   int j, kind = 0x7fffffff;
+   int kind = 0x7fffffff;
    for (;;)
    {
       if (++jjround == 0x7fffffff)
@@ -644,7 +609,7 @@
       if (curChar < 64)
       {
          long l = 1L << curChar;
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -673,7 +638,7 @@
       else if (curChar < 128)
       {
          long l = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -688,7 +653,7 @@
          long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -728,15 +693,7 @@
 {
    return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
 }
-private final int jjStartNfaWithStates_2(int pos, int kind, int state)
-{
-   jjmatchedKind = kind;
-   jjmatchedPos = pos;
-   try { curChar = input_stream.readChar(); }
-   catch(java.io.IOException e) { return pos + 1; }
-   return jjMoveNfa_2(state, pos + 1);
-}
-private final int jjMoveStringLiteralDfa0_2()
+private int jjMoveStringLiteralDfa0_2()
 {
    switch(curChar)
    {
@@ -748,7 +705,7 @@
          return jjMoveNfa_2(0, 0);
    }
 }
-private final int jjMoveStringLiteralDfa1_2(long active0)
+private int jjMoveStringLiteralDfa1_2(long active0)
 {
    try { curChar = input_stream.readChar(); }
    catch(java.io.IOException e) {
@@ -766,14 +723,21 @@
    }
    return jjStartNfa_2(0, active0);
 }
-private final int jjMoveNfa_2(int startState, int curPos)
+private int jjStartNfaWithStates_2(int pos, int kind, int state)
+{
+   jjmatchedKind = kind;
+   jjmatchedPos = pos;
+   try { curChar = input_stream.readChar(); }
+   catch(java.io.IOException e) { return pos + 1; }
+   return jjMoveNfa_2(state, pos + 1);
+}
+private int jjMoveNfa_2(int startState, int curPos)
 {
-   int[] nextStates;
    int startsAt = 0;
    jjnewStateCnt = 5;
    int i = 1;
    jjstateSet[0] = startState;
-   int j, kind = 0x7fffffff;
+   int kind = 0x7fffffff;
    for (;;)
    {
       if (++jjround == 0x7fffffff)
@@ -781,7 +745,7 @@
       if (curChar < 64)
       {
          long l = 1L << curChar;
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -826,7 +790,7 @@
       else if (curChar < 128)
       {
          long l = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -852,7 +816,7 @@
          long l1 = 1L << (hiByte & 077);
          int i2 = (curChar & 0xff) >> 6;
          long l2 = 1L << (curChar & 077);
-         MatchLoop: do
+         do
          {
             switch(jjstateSet[--i])
             {
@@ -895,22 +859,28 @@
    {
       case 0:
          return ((jjbitVec2[i2] & l2) != 0L);
-      default : 
+      default :
          if ((jjbitVec0[i1] & l1) != 0L)
             return true;
          return false;
    }
 }
+
+/** Token literal values. */
 public static final String[] jjstrLiteralImages = {
 "", null, null, null, null, null, null, null, null, null, "\53", "\55", "\50", 
 "\51", "\72", "\136", null, null, null, null, null, "\133", "\173", null, "\124\117", 
 "\135", null, null, "\124\117", "\175", null, null, };
+
+/** Lexer state names. */
 public static final String[] lexStateNames = {
-   "Boost", 
-   "RangeEx", 
-   "RangeIn", 
-   "DEFAULT", 
+   "Boost",
+   "RangeEx",
+   "RangeIn",
+   "DEFAULT",
 };
+
+/** Lex State array. */
 public static final int[] jjnewLexState = {
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, 2, 1, 3, -1, 
    3, -1, -1, -1, 3, -1, -1, 
@@ -925,15 +895,18 @@
 private final int[] jjrounds = new int[33];
 private final int[] jjstateSet = new int[66];
 protected char curChar;
-public PrecedenceQueryParserTokenManager(CharStream stream)
-{
+/** Constructor. */
+public PrecedenceQueryParserTokenManager(CharStream stream){
    input_stream = stream;
 }
-public PrecedenceQueryParserTokenManager(CharStream stream, int lexState)
-{
+
+/** Constructor. */
+public PrecedenceQueryParserTokenManager(CharStream stream, int lexState){
    this(stream);
    SwitchTo(lexState);
 }
+
+/** Reinitialise parser. */
 public void ReInit(CharStream stream)
 {
    jjmatchedPos = jjnewStateCnt = 0;
@@ -941,18 +914,22 @@
    input_stream = stream;
    ReInitRounds();
 }
-private final void ReInitRounds()
+private void ReInitRounds()
 {
    int i;
    jjround = 0x80000001;
    for (i = 33; i-- > 0;)
       jjrounds[i] = 0x80000000;
 }
+
+/** Reinitialise parser. */
 public void ReInit(CharStream stream, int lexState)
 {
    ReInit(stream);
    SwitchTo(lexState);
 }
+
+/** Switch to specified lex state. */
 public void SwitchTo(int lexState)
 {
    if (lexState >= 4 || lexState < 0)
@@ -963,14 +940,25 @@
 
 protected Token jjFillToken()
 {
-   Token t = Token.newToken(jjmatchedKind);
-   t.kind = jjmatchedKind;
+   final Token t;
+   final String curTokenImage;
+   final int beginLine;
+   final int endLine;
+   final int beginColumn;
+   final int endColumn;
    String im = jjstrLiteralImages[jjmatchedKind];
-   t.image = (im == null) ? input_stream.GetImage() : im;
-   t.beginLine = input_stream.getBeginLine();
-   t.beginColumn = input_stream.getBeginColumn();
-   t.endLine = input_stream.getEndLine();
-   t.endColumn = input_stream.getEndColumn();
+   curTokenImage = (im == null) ? input_stream.GetImage() : im;
+   beginLine = input_stream.getBeginLine();
+   beginColumn = input_stream.getBeginColumn();
+   endLine = input_stream.getEndLine();
+   endColumn = input_stream.getEndColumn();
+   t = Token.newToken(jjmatchedKind, curTokenImage);
+
+   t.beginLine = beginLine;
+   t.endLine = endLine;
+   t.beginColumn = beginColumn;
+   t.endColumn = endColumn;
+
    return t;
 }
 
@@ -981,22 +969,21 @@
 int jjmatchedPos;
 int jjmatchedKind;
 
+/** Get the next Token. */
 public Token getNextToken() 
 {
-  int kind;
-  Token specialToken = null;
   Token matchedToken;
   int curPos = 0;
 
   EOFLoop :
   for (;;)
-  {   
-   try   
-   {     
+  {
+   try
+   {
       curChar = input_stream.BeginToken();
-   }     
+   }
    catch(java.io.IOException e)
-   {        
+   {
       jjmatchedKind = 0;
       matchedToken = jjFillToken();
       return matchedToken;
@@ -1066,4 +1053,31 @@
   }
 }
 
+private void jjCheckNAdd(int state)
+{
+   if (jjrounds[state] != jjround)
+   {
+      jjstateSet[jjnewStateCnt++] = state;
+      jjrounds[state] = jjround;
+   }
+}
+private void jjAddStates(int start, int end)
+{
+   do {
+      jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+   } while (start++ != end);
+}
+private void jjCheckNAddTwoStates(int state1, int state2)
+{
+   jjCheckNAdd(state1);
+   jjCheckNAdd(state2);
+}
+
+private void jjCheckNAddStates(int start, int end)
+{
+   do {
+      jjCheckNAdd(jjnextStates[start]);
+   } while (start++ != end);
+}
+
 }



Mime
View raw message