lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r824116 [2/2] - in /lucene/java/trunk: ./ contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/m...
Date Sun, 11 Oct 2009 17:35:11 GMT
Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/Token.java Sun Oct 11 17:35:09 2009
@@ -1,4 +1,5 @@
-/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 4.1 */
+/* JavaCCOptions:TOKEN_EXTENDS=,KEEP_LINE_COL=null */
 package org.apache.lucene.queryParser.precedence;
 
 /**
@@ -14,12 +15,14 @@
    */
   public int kind;
 
-  /**
-   * beginLine and beginColumn describe the position of the first character
-   * of this token; endLine and endColumn describe the position of the
-   * last character of this token.
-   */
-  public int beginLine, beginColumn, endLine, endColumn;
+  /** The line number of the first character of this Token. */
+  public int beginLine;
+  /** The column number of the first character of this Token. */
+  public int beginColumn;
+  /** The line number of the last character of this Token. */
+  public int endLine;
+  /** The column number of the last character of this Token. */
+  public int endColumn;
 
   /**
    * The string image of the token.
@@ -51,6 +54,40 @@
   public Token specialToken;
 
   /**
+   * An optional attribute value of the Token.
+   * Tokens which are not used as syntactic sugar will often contain
+   * meaningful values that will be used later on by the compiler or
+   * interpreter. This attribute value is often different from the image.
+   * Any subclass of Token that actually wants to return a non-null value can
+   * override this method as appropriate.
+   */
+  public Object getValue() {
+    return null;
+  }
+
+  /**
+   * No-argument constructor
+   */
+  public Token() {}
+
+  /**
+   * Constructs a new token for the specified Image.
+   */
+  public Token(int kind)
+  {
+     this(kind, null);
+  }
+
+  /**
+   * Constructs a new token for the specified Image and Kind.
+   */
+  public Token(int kind, String image)
+  {
+     this.kind = kind;
+     this.image = image;
+  }
+
+  /**
    * Returns the image.
    */
   public String toString()
@@ -63,19 +100,25 @@
    * can create and return subclass objects based on the value of ofKind.
    * Simply add the cases to the switch for all those special cases.
    * For example, if you have a subclass of Token called IDToken that
-   * you want to create if ofKind is ID, simlpy add something like :
+   * you want to create if ofKind is ID, simply add something like :
    *
-   *    case MyParserConstants.ID : return new IDToken();
+   *    case MyParserConstants.ID : return new IDToken(ofKind, image);
    *
    * to the following switch statement. Then you can cast matchedToken
-   * variable to the appropriate type and use it in your lexical actions.
+   * variable to the appropriate type and use sit in your lexical actions.
    */
-  public static final Token newToken(int ofKind)
+  public static Token newToken(int ofKind, String image)
   {
      switch(ofKind)
      {
-       default : return new Token();
+       default : return new Token(ofKind, image);
      }
   }
 
+  public static Token newToken(int ofKind)
+  {
+     return newToken(ofKind, null);
+  }
+
 }
+/* JavaCC - OriginalChecksum=0dc5808f2ab8aac8775ea9175fa2cb51 (do not edit this line) */

Modified: lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java (original)
+++ lucene/java/trunk/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/TokenMgrError.java Sun Oct 11 17:35:09 2009
@@ -1,19 +1,22 @@
-/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 4.1 */
+/* JavaCCOptions: */
 package org.apache.lucene.queryParser.precedence;
 
+/** Token Manager Error. */
 public class TokenMgrError extends Error
 {
+
    /*
     * Ordinals for various reasons why an Error of this type can be thrown.
     */
 
    /**
-    * Lexical error occured.
+    * Lexical error occurred.
     */
    static final int LEXICAL_ERROR = 0;
 
    /**
-    * An attempt wass made to create a second instance of a static token manager.
+    * An attempt was made to create a second instance of a static token manager.
     */
    static final int STATIC_LEXER_ERROR = 1;
 
@@ -34,7 +37,7 @@
    int errorCode;
 
    /**
-    * Replaces unprintable characters by their espaced (or unicode escaped)
+    * Replaces unprintable characters by their escaped (or unicode escaped)
     * equivalents in the given string
     */
    protected static final String addEscapes(String str) {
@@ -85,12 +88,12 @@
    /**
     * Returns a detailed message for the Error when it is thrown by the
     * token manager to indicate a lexical error.
-    * Parameters : 
-    *    EOFSeen     : indicates if EOF caused the lexicl error
-    *    curLexState : lexical state in which this error occured
-    *    errorLine   : line number when the error occured
-    *    errorColumn : column number when the error occured
-    *    errorAfter  : prefix that was seen before this error occured
+    * Parameters :
+    *    EOFSeen     : indicates if EOF caused the lexical error
+    *    curLexState : lexical state in which this error occurred
+    *    errorLine   : line number when the error occurred
+    *    errorColumn : column number when the error occurred
+    *    errorAfter  : prefix that was seen before this error occurred
     *    curchar     : the offending character
     * Note: You can customize the lexical error message by modifying this method.
     */
@@ -105,7 +108,7 @@
    /**
     * You can also modify the body of this method to customize your error messages.
     * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
-    * of end-users concern, so you can return something like : 
+    * of end-users concern, so you can return something like :
     *
     *     "Internal Error : Please file a bug report .... "
     *
@@ -119,15 +122,19 @@
     * Constructors of various flavors follow.
     */
 
+   /** No arg constructor. */
    public TokenMgrError() {
    }
 
+   /** Constructor with message and reason. */
    public TokenMgrError(String message, int reason) {
       super(message);
       errorCode = reason;
    }
 
+   /** Full Constructor. */
    public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
       this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
    }
 }
+/* JavaCC - OriginalChecksum=43f834e732189cec173ab002c94ae97f (do not edit this line) */

Modified: lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java (original)
+++ lucene/java/trunk/contrib/misc/src/test/org/apache/lucene/queryParser/analyzing/TestAnalyzingQueryParser.java Sun Oct 11 17:35:09 2009
@@ -22,7 +22,7 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.ISOLatin1AccentFilter;
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
 import org.apache.lucene.analysis.LowerCaseFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardFilter;
@@ -111,7 +111,7 @@
   public TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new StandardTokenizer(reader);
     result = new StandardFilter(result);
-    result = new ISOLatin1AccentFilter(result);
+    result = new ASCIIFoldingFilter(result);
     result = new LowerCaseFilter(result);
     return result;
   }

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java Sun Oct 11 17:35:09 2009
@@ -181,48 +181,6 @@
 
     }
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.lucene.analysis.TokenStream#next(org.apache.lucene.analysis
-     * .Token)
-     */
-    @Override
-    public Token next(Token reusableToken) throws IOException {
-
-      if (multiToken > 0) {
-        reusableToken.setTermBuffer("multi" + (multiToken + 1));
-        reusableToken.setStartOffset(prevStartOffset);
-        reusableToken.setEndOffset(prevEndOffset);
-        reusableToken.setType(prevType);
-        reusableToken.setPositionIncrement(0);
-        multiToken--;
-        return reusableToken;
-      } else {
-        boolean next = (reusableToken = input.next(token)) != null;
-        if (next == false) {
-          return null;
-        }
-        prevType = reusableToken.type();
-        prevStartOffset = reusableToken.startOffset();
-        prevEndOffset = reusableToken.endOffset();
-        String text = reusableToken.term();
-        if (text.equals("triplemulti")) {
-          multiToken = 2;
-          return reusableToken;
-        } else if (text.equals("multi")) {
-          multiToken = 1;
-          return reusableToken;
-        } else {
-          return reusableToken;
-        }
-      }
-
-    }
-
-    private Token token = new Token();
-
     public final boolean incrementToken() throws java.io.IOException {
       if (multiToken > 0) {
         termAtt.setTermBuffer("multi" + (multiToken + 1));
@@ -282,30 +240,6 @@
       posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     }
 
-    private Token token = new Token();
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see org.apache.lucene.analysis.TokenStream#next()
-     */
-    @Override
-    public Token next(Token reusableToken) throws IOException {
-      while (null != (reusableToken = input.next(token))) {
-        String term = reusableToken.term();
-        if (term.equals("the")) {
-          // stopword, do nothing
-        } else if (term.equals("quick")) {
-          reusableToken.setPositionIncrement(2);
-          return reusableToken;
-        } else {
-          reusableToken.setPositionIncrement(1);
-          return reusableToken;
-        }
-      }
-      return null;
-    }
-
     public final boolean incrementToken() throws java.io.IOException {
       while (input.incrementToken()) {
         if (termAtt.term().equals("the")) {

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java Sun Oct 11 17:35:09 2009
@@ -175,48 +175,6 @@
 
     }
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.lucene.analysis.TokenStream#next(org.apache.lucene.analysis
-     * .Token)
-     */
-    @Override
-    public Token next(Token reusableToken) throws IOException {
-
-      if (multiToken > 0) {
-        reusableToken.setTermBuffer("multi" + (multiToken + 1));
-        reusableToken.setStartOffset(prevStartOffset);
-        reusableToken.setEndOffset(prevEndOffset);
-        reusableToken.setType(prevType);
-        reusableToken.setPositionIncrement(0);
-        multiToken--;
-        return reusableToken;
-      } else {
-        boolean next = (reusableToken = input.next(token)) != null;
-        if (next == false) {
-          return null;
-        }
-        prevType = reusableToken.type();
-        prevStartOffset = reusableToken.startOffset();
-        prevEndOffset = reusableToken.endOffset();
-        String text = reusableToken.term();
-        if (text.equals("triplemulti")) {
-          multiToken = 2;
-          return reusableToken;
-        } else if (text.equals("multi")) {
-          multiToken = 1;
-          return reusableToken;
-        } else {
-          return reusableToken;
-        }
-      }
-
-    }
-
-    private Token token = new Token();
-
     public final boolean incrementToken() throws java.io.IOException {
       if (multiToken > 0) {
         termAtt.setTermBuffer("multi" + (multiToken + 1));
@@ -276,30 +234,6 @@
       posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     }
 
-    private Token token = new Token();
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see org.apache.lucene.analysis.TokenStream#next()
-     */
-    @Override
-    public Token next(Token reusableToken) throws IOException {
-      while (null != (reusableToken = input.next(token))) {
-        String term = reusableToken.term();
-        if (term.equals("the")) {
-          // stopword, do nothing
-        } else if (term.equals("quick")) {
-          reusableToken.setPositionIncrement(2);
-          return reusableToken;
-        } else {
-          reusableToken.setPositionIncrement(1);
-          return reusableToken;
-        }
-      }
-      return null;
-    }
-
     public final boolean incrementToken() throws java.io.IOException {
       while (input.incrementToken()) {
         if (termAtt.term().equals("the")) {

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQPHelper.java Sun Oct 11 17:35:09 2009
@@ -359,8 +359,8 @@
     }
 
     private static class EmptyTokenStream extends TokenStream {
-      public Token next(final Token reusableToken) {
-        return null;
+      public boolean incrementToken() {
+        return false;
       }
     }
   }

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiFieldQueryParserWrapper.java Sun Oct 11 17:35:09 2009
@@ -357,8 +357,8 @@
     }
 
     private static class EmptyTokenStream extends TokenStream {
-      public Token next(final Token reusableToken) {
-        return null;
+      public boolean incrementToken() {
+        return false;
       }
     }
   }

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Sun Oct 11 17:35:09 2009
@@ -113,33 +113,6 @@
     boolean inPhrase = false;
     int savedStart = 0, savedEnd = 0;
 
-    public Token next(Token reusableToken) throws IOException {
-      Token token = reusableToken;
-
-      if (inPhrase) {
-        inPhrase = false;
-        token.setTermBuffer("phrase2");
-        token.setStartOffset(savedStart);
-        token.setEndOffset(savedEnd);
-        return reusableToken;
-      } else
-        while ((token = this.input.next(reusableToken)) != null) {
-          if (token.term().equals("phrase")) {
-            inPhrase = true;
-            savedStart = token.startOffset();
-            savedEnd = token.endOffset();
-            token.setTermBuffer("phrase1");
-            token.setStartOffset(savedStart);
-            token.setEndOffset(savedEnd);
-            return token;
-          } else if (!token.term().equals("stop"))
-            return token;
-        }
-
-      return null;
-
-    }
-
     public boolean incrementToken() throws IOException {
       if (inPhrase) {
         inPhrase = false;
@@ -1185,12 +1158,12 @@
 
   private class CannedTokenStream extends TokenStream {
     private int upto = 0;
+    final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
+    final TermAttribute term = addAttribute(TermAttribute.class);
     public boolean incrementToken() {
       if (upto == 4) {
         return false;
       }
-      PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
-      TermAttribute term = addAttribute(TermAttribute.class);
       if (upto == 0) {
         posIncr.setPositionIncrement(1);
         term.setTermBuffer("a");

Modified: lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original)
+++ lucene/java/trunk/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Sun Oct 11 17:35:09 2009
@@ -108,33 +108,6 @@
     boolean inPhrase = false;
     int savedStart = 0, savedEnd = 0;
 
-    public Token next(Token reusableToken) throws IOException {
-      Token token = reusableToken;
-
-      if (inPhrase) {
-        inPhrase = false;
-        token.setTermBuffer("phrase2");
-        token.setStartOffset(savedStart);
-        token.setEndOffset(savedEnd);
-        return reusableToken;
-      } else
-        while ((token = this.input.next(reusableToken)) != null) {
-          if (token.term().equals("phrase")) {
-            inPhrase = true;
-            savedStart = token.startOffset();
-            savedEnd = token.endOffset();
-            token.setTermBuffer("phrase1");
-            token.setStartOffset(savedStart);
-            token.setEndOffset(savedEnd);
-            return token;
-          } else if (!token.term().equals("stop"))
-            return token;
-        }
-
-      return null;
-
-    }
-
     public boolean incrementToken() throws IOException {
       if (inPhrase) {
         inPhrase = false;

Modified: lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java (original)
+++ lucene/java/trunk/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java Sun Oct 11 17:35:09 2009
@@ -30,7 +30,7 @@
  *
  * Available stemmers are listed in {@link org.tartarus.snowball.ext}.
  */
-public class SnowballFilter extends TokenFilter {
+public final class SnowballFilter extends TokenFilter {
 
   private SnowballProgram stemmer;
 
@@ -78,16 +78,4 @@
       return false;
     }
   }
-  
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
 }

Modified: lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java (original)
+++ lucene/java/trunk/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java Sun Oct 11 17:35:09 2009
@@ -39,7 +39,7 @@
  * EXPERIMENTAL !!!!!!!!!
  * NOTE: This Tokenizer is considered experimental and the grammar is subject to change in the trunk and in follow up releases.
  */
-public class WikipediaTokenizer extends Tokenizer {
+public final class WikipediaTokenizer extends Tokenizer {
   public static final String INTERNAL_LINK = "il";
   public static final String EXTERNAL_LINK = "el";
   //The URL part of the link, i.e. the first token
@@ -187,18 +187,6 @@
     this.termAtt = addAttribute(TermAttribute.class);
     this.flagsAtt = addAttribute(FlagsAttribute.class);    
   }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
   
   /*
   * (non-Javadoc)

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/CachingTokenFilter.java Sun Oct 11 17:35:09 2009
@@ -33,7 +33,7 @@
  * {@link TokenStream#reset()}, which repositions the
  * stream to the first Token. 
  */
-public class CachingTokenFilter extends TokenFilter {
+public final class CachingTokenFilter extends TokenFilter {
   private List cache = null;
   private Iterator iterator = null; 
   private AttributeSource.State finalState;
@@ -41,18 +41,6 @@
   public CachingTokenFilter(TokenStream input) {
     super(input);
   }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws IOException {
-    return super.next();
-  }
   
   public final boolean incrementToken() throws IOException {
     if (cache == null) {

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java Sun Oct 11 17:35:09 2009
@@ -114,18 +114,6 @@
     offsetAtt.setOffset(finalOffset, finalOffset);
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws IOException {
-    return super.next();
-  }
-
   public void reset(Reader input) throws IOException {
     super.reset(input);
     bufferIndex = 0;

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java Sun Oct 11 17:35:09 2009
@@ -26,10 +26,12 @@
  * For instance, 'à' will be replaced by 'a'.
  * <p>
  * 
- * @deprecated in favor of {@link ASCIIFoldingFilter} which covers a superset 
- * of Latin 1. This class will be removed in Lucene 3.0.
+ * @deprecated If you build a new index, use {@link ASCIIFoldingFilter}
+ * which covers a superset of Latin 1.
+ * This class is included for use with existing
+ * indexes and will be removed in a future release (possibly Lucene 4.0).
  */
-public class ISOLatin1AccentFilter extends TokenFilter {
+public final class ISOLatin1AccentFilter extends TokenFilter {
   public ISOLatin1AccentFilter(TokenStream input) {
     super(input);
     termAtt = addAttribute(TermAttribute.class);
@@ -58,18 +60,6 @@
       return false;
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws java.io.IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws java.io.IOException {
-    return super.next();
-  }
-
   /**
    * To replace accented characters in a String by unaccented equivalents.
    */

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java Sun Oct 11 17:35:09 2009
@@ -27,7 +27,7 @@
 /**
  * Emits the entire input as a single token.
  */
-public class KeywordTokenizer extends Tokenizer {
+public final class KeywordTokenizer extends Tokenizer {
   
   private static final int DEFAULT_BUFFER_SIZE = 256;
 
@@ -88,18 +88,6 @@
     offsetAtt.setOffset(finalOffset, finalOffset);
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws IOException {
-    return super.next();
-  }
-
   public void reset(Reader input) throws IOException {
     super.reset(input);
     this.done = false;

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/Token.java Sun Oct 11 17:35:09 2009
@@ -867,9 +867,6 @@
       if (payload !=null) {
         to.payload = (Payload) payload.clone();
       }
-    // remove the following optimization in 3.0 when old TokenStream API removed:
-    } else if (target instanceof TokenWrapper) {
-      ((TokenWrapper) target).delegate = (Token) this.clone();
     } else {
       initTermBuffer();
       ((TermAttribute) target).setTermBuffer(termBuffer, 0, termLength);

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java Sun Oct 11 17:35:09 2009
@@ -18,18 +18,10 @@
  */
 
 import java.io.IOException;
-import java.util.IdentityHashMap;
 
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Payload;
 import org.apache.lucene.util.Attribute;
 import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.util.AttributeSource;
@@ -78,103 +70,19 @@
  * You can find some example code for the new API in the analysis package level
  * Javadoc.
  * <p>
- * Sometimes it is desirable to capture a current state of a <code>TokenStream</code>
- * , e. g. for buffering purposes (see {@link CachingTokenFilter},
+ * Sometimes it is desirable to capture a current state of a <code>TokenStream</code>,
+ * e.g. for buffering purposes (see {@link CachingTokenFilter},
  * {@link TeeSinkTokenFilter}). For this usecase
  * {@link AttributeSource#captureState} and {@link AttributeSource#restoreState}
  * can be used.
  */
 public abstract class TokenStream extends AttributeSource {
 
-  /** @deprecated Remove this when old API is removed! */
-  private static final AttributeFactory DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
-    = new TokenWrapperAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
-  
-  /** @deprecated Remove this when old API is removed! */
-  private final TokenWrapper tokenWrapper;
-  
-  /** @deprecated Remove this when old API is removed! */
-  private static boolean onlyUseNewAPI = false;
-
-  /** @deprecated Remove this when old API is removed! */
-  private final MethodSupport supportedMethods = getSupportedMethods(this.getClass());
-
-  /** @deprecated Remove this when old API is removed! */
-  private static final class MethodSupport {
-    final boolean hasIncrementToken, hasReusableNext, hasNext;
-
-    MethodSupport(Class<? extends TokenStream> clazz) {
-      hasIncrementToken = isMethodOverridden(clazz, "incrementToken");
-      hasReusableNext = isMethodOverridden(clazz, "next", Token.class);
-      hasNext = isMethodOverridden(clazz, "next");
-    }
-    
-    private static boolean isMethodOverridden(Class<? extends TokenStream> clazz, String name, Class... params) {
-      try {
-        return clazz.getMethod(name, params).getDeclaringClass() != TokenStream.class;
-      } catch (NoSuchMethodException e) {
-        // should not happen
-        throw new RuntimeException(e);
-      }
-    }
-  }
-      
-  /** @deprecated Remove this when old API is removed! */
-  private static final IdentityHashMap<Class<? extends TokenStream>,MethodSupport> knownMethodSupport =
-    new IdentityHashMap<Class<? extends TokenStream>,MethodSupport>();
-  
-  /** @deprecated Remove this when old API is removed! */
-  private static MethodSupport getSupportedMethods(Class<? extends TokenStream> clazz) {
-    MethodSupport supportedMethods;
-    synchronized(knownMethodSupport) {
-      supportedMethods = knownMethodSupport.get(clazz);
-      if (supportedMethods == null) {
-        knownMethodSupport.put(clazz, supportedMethods = new MethodSupport(clazz));
-      }
-    }
-    return supportedMethods;
-  }
-
-  /** @deprecated Remove this when old API is removed! */
-  private static final class TokenWrapperAttributeFactory extends AttributeFactory {
-    private final AttributeFactory delegate;
-  
-    private TokenWrapperAttributeFactory(AttributeFactory delegate) {
-      this.delegate = delegate;
-    }
-  
-    public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
-      return attClass.isAssignableFrom(TokenWrapper.class)
-        ? new TokenWrapper()
-        : delegate.createAttributeInstance(attClass);
-    }
-    
-    // this is needed for TeeSinkTokenStream's check for compatibility of AttributeSource,
-    // so two TokenStreams using old API have the same AttributeFactory wrapped by this one.
-    public boolean equals(Object other) {
-      if (this == other) return true;
-      if (other instanceof TokenWrapperAttributeFactory) {
-        final TokenWrapperAttributeFactory af = (TokenWrapperAttributeFactory) other;
-        return this.delegate.equals(af.delegate);
-      }
-      return false;
-    }
-    
-    public int hashCode() {
-      return delegate.hashCode() ^ 0x0a45ff31;
-    }
-  }
-
   /**
    * A TokenStream using the default attribute factory.
    */
   protected TokenStream() {
-    super(onlyUseNewAPI
-      ? AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY
-      : TokenStream.DEFAULT_TOKEN_WRAPPER_ATTRIBUTE_FACTORY
-    );
-    tokenWrapper = initTokenWrapper(null);
-    check();
+    super();
   }
   
   /**
@@ -182,102 +90,13 @@
    */
   protected TokenStream(AttributeSource input) {
     super(input);
-    tokenWrapper = initTokenWrapper(input);
-    check();
   }
   
   /**
    * A TokenStream using the supplied AttributeFactory for creating new {@link Attribute} instances.
    */
   protected TokenStream(AttributeFactory factory) {
-    super(onlyUseNewAPI
-      ? factory
-      : new TokenWrapperAttributeFactory(factory)
-    );
-    tokenWrapper = initTokenWrapper(null);
-    check();
-  }
-
-  /** @deprecated Remove this when old API is removed! */
-  private TokenWrapper initTokenWrapper(AttributeSource input) {
-    if (onlyUseNewAPI) {
-      // no wrapper needed
-      return null;
-    } else {
-      // if possible get the wrapper from the filter's input stream
-      if (input instanceof TokenStream && ((TokenStream) input).tokenWrapper != null) {
-        return ((TokenStream) input).tokenWrapper;
-      }
-      // check that all attributes are implemented by the same TokenWrapper instance
-      final Attribute att = addAttribute(TermAttribute.class);
-      if (att instanceof TokenWrapper &&
-        addAttribute(TypeAttribute.class) == att &&
-        addAttribute(PositionIncrementAttribute.class) == att &&
-        addAttribute(FlagsAttribute.class) == att &&
-        addAttribute(OffsetAttribute.class) == att &&
-        addAttribute(PayloadAttribute.class) == att
-      ) {
-        return (TokenWrapper) att;
-      } else {
-        throw new UnsupportedOperationException(
-          "If onlyUseNewAPI is disabled, all basic Attributes must be implemented by the internal class "+
-          "TokenWrapper. Please make sure, that all TokenStreams/TokenFilters in this chain have been "+
-          "instantiated with this flag disabled and do not add any custom instances for the basic Attributes!"
-        );
-      }
-    }
-  }
-
-  /** @deprecated Remove this when old API is removed! */
-  private void check() {
-    if (onlyUseNewAPI && !supportedMethods.hasIncrementToken) {
-      throw new UnsupportedOperationException(getClass().getName()+" does not implement incrementToken() which is needed for onlyUseNewAPI.");
-    }
-
-    // a TokenStream subclass must at least implement one of the methods!
-    if (!(supportedMethods.hasIncrementToken || supportedMethods.hasNext || supportedMethods.hasReusableNext)) {
-      throw new UnsupportedOperationException(getClass().getName()+" does not implement any of incrementToken(), next(Token), next().");
-    }
-  }
-  
-  /**
-   * For extra performance you can globally enable the new
-   * {@link #incrementToken} API using {@link Attribute}s. There will be a
-   * small, but in most cases negligible performance increase by enabling this,
-   * but it only works if <b>all</b> <code>TokenStream</code>s use the new API and
-   * implement {@link #incrementToken}. This setting can only be enabled
-   * globally.
-   * <P>
-   * This setting only affects <code>TokenStream</code>s instantiated after this
-   * call. All <code>TokenStream</code>s already created use the other setting.
-   * <P>
-   * All core {@link Analyzer}s are compatible with this setting, if you have
-   * your own <code>TokenStream</code>s that are also compatible, you should enable
-   * this.
-   * <P>
-   * When enabled, tokenization may throw {@link UnsupportedOperationException}
-   * s, if the whole tokenizer chain is not compatible eg one of the
-   * <code>TokenStream</code>s does not implement the new <code>TokenStream</code> API.
-   * <P>
-   * The default is <code>false</code>, so there is the fallback to the old API
-   * available.
-   * 
-   * @deprecated This setting will no longer be needed in Lucene 3.0 as the old
-   *             API will be removed.
-   */
-  public static void setOnlyUseNewAPI(boolean onlyUseNewAPI) {
-    TokenStream.onlyUseNewAPI = onlyUseNewAPI;
-  }
-  
-  /**
-   * Returns if only the new API is used.
-   * 
-   * @see #setOnlyUseNewAPI
-   * @deprecated This setting will no longer be needed in Lucene 3.0 as
-   *             the old API will be removed.
-   */
-  public static boolean getOnlyUseNewAPI() {
-    return onlyUseNewAPI;
+    super(factory);
   }
   
   /**
@@ -293,7 +112,7 @@
    * <p>
    * This method is called for every token of a document, so an efficient
    * implementation is crucial for good performance. To avoid calls to
-   * {@link #addAttribute(Class)} and {@link #getAttribute(Class)} or downcasts,
+   * {@link #addAttribute(Class)} and {@link #getAttribute(Class)},
    * references to all {@link AttributeImpl}s that this stream uses should be
    * retrieved during instantiation.
    * <p>
@@ -303,25 +122,8 @@
    * {@link #incrementToken()}.
    * 
    * @return false for end of stream; true otherwise
-   * 
-   *         <p>
-   *         <b>Note that this method will be defined abstract in Lucene
-   *         3.0.</b>
    */
-  public boolean incrementToken() throws IOException {
-    assert tokenWrapper != null;
-    
-    final Token token;
-    if (supportedMethods.hasReusableNext) {
-      token = next(tokenWrapper.delegate);
-    } else {
-      assert supportedMethods.hasNext;
-      token = next();
-    }
-    if (token == null) return false;
-    tokenWrapper.delegate = token;
-    return true;
-  }
+  public abstract boolean incrementToken() throws IOException;
   
   /**
    * This method is called by the consumer after the last token has been
@@ -341,83 +143,6 @@
   }
 
   /**
-   * Returns the next token in the stream, or null at EOS. When possible, the
-   * input Token should be used as the returned Token (this gives fastest
-   * tokenization performance), but this is not required and a new Token may be
-   * returned. Callers may re-use a single Token instance for successive calls
-   * to this method.
-   * <p>
-   * This implicitly defines a "contract" between consumers (callers of this
-   * method) and producers (implementations of this method that are the source
-   * for tokens):
-   * <ul>
-   * <li>A consumer must fully consume the previously returned {@link Token}
-   * before calling this method again.</li>
-   * <li>A producer must call {@link Token#clear()} before setting the fields in
-   * it and returning it</li>
-   * </ul>
-   * Also, the producer must make no assumptions about a {@link Token} after it
-   * has been returned: the caller may arbitrarily change it. If the producer
-   * needs to hold onto the {@link Token} for subsequent calls, it must clone()
-   * it before storing it. Note that a {@link TokenFilter} is considered a
-   * consumer.
-   * 
-   * @param reusableToken a {@link Token} that may or may not be used to return;
-   *        this parameter should never be null (the callee is not required to
-   *        check for null before using it, but it is a good idea to assert that
-   *        it is not null.)
-   * @return next {@link Token} in the stream or null if end-of-stream was hit
-   * @deprecated The new {@link #incrementToken()} and {@link AttributeSource}
-   *             APIs should be used instead.
-   */
-  public Token next(final Token reusableToken) throws IOException {
-    assert reusableToken != null;
-    
-    if (tokenWrapper == null)
-      throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-    
-    if (supportedMethods.hasIncrementToken) {
-      tokenWrapper.delegate = reusableToken;
-      return incrementToken() ? tokenWrapper.delegate : null;
-    } else {
-      assert supportedMethods.hasNext;
-      return next();
-    }
-  }
-
-  /**
-   * Returns the next {@link Token} in the stream, or null at EOS.
-   * 
-   * @deprecated The returned Token is a "full private copy" (not re-used across
-   *             calls to {@link #next()}) but will be slower than calling
-   *             {@link #next(Token)} or using the new {@link #incrementToken()}
-   *             method with the new {@link AttributeSource} API.
-   */
-  public Token next() throws IOException {
-    if (tokenWrapper == null)
-      throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
-    
-    final Token nextToken;
-    if (supportedMethods.hasIncrementToken) {
-      final Token savedDelegate = tokenWrapper.delegate;
-      tokenWrapper.delegate = new Token();
-      nextToken = incrementToken() ? tokenWrapper.delegate : null;
-      tokenWrapper.delegate = savedDelegate;
-    } else {
-      assert supportedMethods.hasReusableNext;
-      nextToken = next(new Token());
-    }
-    
-    if (nextToken != null) {
-      Payload p = nextToken.getPayload();
-      if (p != null) {
-        nextToken.setPayload((Payload) p.clone());
-      }
-    }
-    return nextToken;
-  }
-
-  /**
    * Resets this stream to the beginning. This is an optional operation, so
    * subclasses may or may not implement this method. {@link #reset()} is not needed for
    * the standard indexing process. However, if the tokens of a

Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Sun Oct 11 17:35:09 2009
@@ -45,7 +45,7 @@
  * directory to your project and maintaining your own grammar-based tokenizer.
  */
 
-public class StandardTokenizer extends Tokenizer {
+public final class StandardTokenizer extends Tokenizer {
   /** A private instance of the JFlex-constructed scanner */
   private final StandardTokenizerImpl scanner;
 
@@ -209,18 +209,6 @@
     offsetAtt.setOffset(finalOffset, finalOffset);
   }
 
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next(final Token reusableToken) throws IOException {
-    return super.next(reusableToken);
-  }
-
-  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
-   * not be overridden. Delegates to the backwards compatibility layer. */
-  public final Token next() throws IOException {
-    return super.next();
-  }
-
   /*
    * (non-Javadoc)
    *

Propchange: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (added)
+++ svn:mergeinfo Sun Oct 11 17:35:09 2009
@@ -0,0 +1 @@
+/lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824

Modified: lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/analysis/TestToken.java Sun Oct 11 17:35:09 2009
@@ -18,8 +18,12 @@
  */
 
 import org.apache.lucene.index.Payload;
-import org.apache.lucene.analysis.tokenattributes.TestSimpleAttributeImpls;
+import org.apache.lucene.analysis.tokenattributes.*;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+
+import java.io.StringReader;
 
 public class TestToken extends LuceneTestCase {
 
@@ -219,4 +223,33 @@
     assertEquals(pl, copy.getPayload());
     assertNotSame(pl, copy.getPayload());
   }
+  
+  public interface SenselessAttribute extends Attribute {}
+  
+  public static final class SenselessAttributeImpl extends AttributeImpl implements SenselessAttribute {
+    public void copyTo(AttributeImpl target) {}
+    public void clear() {}
+    public boolean equals(Object o) { return (o instanceof SenselessAttributeImpl); }
+    public int hashCode() { return 0; }
+  }
+
+  public void testTokenAttributeFactory() throws Exception {
+    TokenStream ts = new WhitespaceTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo bar"));
+    
+    assertTrue("TypeAttribute is not implemented by SenselessAttributeImpl",
+      ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
+    
+    assertTrue("TermAttribute is not implemented by Token",
+      ts.addAttribute(TermAttribute.class) instanceof Token);
+    assertTrue("OffsetAttribute is not implemented by Token",
+      ts.addAttribute(OffsetAttribute.class) instanceof Token);
+    assertTrue("FlagsAttribute is not implemented by Token",
+      ts.addAttribute(FlagsAttribute.class) instanceof Token);
+    assertTrue("PayloadAttribute is not implemented by Token",
+      ts.addAttribute(PayloadAttribute.class) instanceof Token);
+    assertTrue("PositionIncrementAttribute is not implemented by Token", 
+      ts.addAttribute(PositionIncrementAttribute.class) instanceof Token);
+    assertTrue("TypeAttribute is not implemented by Token",
+      ts.addAttribute(TypeAttribute.class) instanceof Token);
+  }
 }

Modified: lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=824116&r1=824115&r2=824116&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java Sun Oct 11 17:35:09 2009
@@ -125,4 +125,20 @@
     assertEquals("Token should only printed once", "("+tok.toString()+")", src.toString());
   }
   
+  public void testDefaultAttributeFactory() throws Exception {
+    AttributeSource src = new AttributeSource();
+    
+    assertTrue("TermAttribute is not implemented by TermAttributeImpl",
+      src.addAttribute(TermAttribute.class) instanceof TermAttributeImpl);
+    assertTrue("OffsetAttribute is not implemented by OffsetAttributeImpl",
+      src.addAttribute(OffsetAttribute.class) instanceof OffsetAttributeImpl);
+    assertTrue("FlagsAttribute is not implemented by FlagsAttributeImpl",
+      src.addAttribute(FlagsAttribute.class) instanceof FlagsAttributeImpl);
+    assertTrue("PayloadAttribute is not implemented by PayloadAttributeImpl",
+      src.addAttribute(PayloadAttribute.class) instanceof PayloadAttributeImpl);
+    assertTrue("PositionIncrementAttribute is not implemented by PositionIncrementAttributeImpl", 
+      src.addAttribute(PositionIncrementAttribute.class) instanceof PositionIncrementAttributeImpl);
+    assertTrue("TypeAttribute is not implemented by TypeAttributeImpl",
+      src.addAttribute(TypeAttribute.class) instanceof TypeAttributeImpl);
+  }
 }



Mime
View raw message