lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From uschind...@apache.org
Subject svn commit: r932749 [1/2] - in /lucene/dev/trunk/lucene: ./ backwards/src/test/org/apache/lucene/analysis/ src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/analysis/standard/ src/java/org/apache/lucene/collation/ src/java/org/apache/luce...
Date Sat, 10 Apr 2010 15:41:28 GMT
Author: uschindler
Date: Sat Apr 10 15:41:27 2010
New Revision: 932749

URL: http://svn.apache.org/viewvc?rev=932749&view=rev
Log:
LUCENE-2372: Convert core analyzers to CharTermAttribute. Also made rest of core analyzers final.

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestAnalyzers.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordMarkerTokenFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/StopFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/collation/CollationKeyFilter.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
    lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestAnalyzers.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestKeywordMarkerTokenFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestToken.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java
    lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sat Apr 10 15:41:27 2010
@@ -97,6 +97,10 @@ Changes in backwards compatibility polic
   TODO: Point to new attribute inspection API coming with LUCENE-2374.
   (Uwe Schindler, Robert Muir)
 
+* LUCENE-2372: StandardAnalyzer, KeywordAnalyzer, PerFieldAnalyzerWrapper
+  are now final.  Also removed the now obsolete and deprecated
+  Analyzer.setOverridesTokenStreamMethod().  (Uwe Schindler)
+
 Changes in runtime behavior
 
 * LUCENE-1923: Made IndexReader.toString() produce something

Modified: lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/dev/trunk/lucene/backwards/src/test/org/apache/lucene/analysis/TestAnalyzers.java Sat Apr 10 15:41:27 2010
@@ -120,6 +120,7 @@ public class TestAnalyzers extends BaseT
     String[] y = StandardTokenizer.TOKEN_TYPES;
   }
 
+  /* StandardAnalyzer was made final in 3.1:
   private static class MyStandardAnalyzer extends StandardAnalyzer {
     public MyStandardAnalyzer() {
       super(org.apache.lucene.util.Version.LUCENE_CURRENT);
@@ -139,6 +140,7 @@ public class TestAnalyzers extends BaseT
     assertTrue(ts.incrementToken());
     assertFalse(ts.incrementToken());
   }
+  */
 }
 
 class PayloadSetter extends TokenFilter {

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java Sat Apr 10 15:41:27 2010
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.RamUsageEstimator;
 
@@ -61,18 +61,17 @@ public final class ASCIIFoldingFilter ex
   public ASCIIFoldingFilter(TokenStream input)
   {
     super(input);
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   private char[] output = new char[512];
   private int outputPos;
-  private TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final char[] buffer = termAtt.termBuffer();
-      final int length = termAtt.termLength();
+      final char[] buffer = termAtt.buffer();
+      final int length = termAtt.length();
 
       // If no characters actually require rewriting then we
       // just return token as-is:
@@ -81,7 +80,7 @@ public final class ASCIIFoldingFilter ex
         if (c >= '\u0080')
         {
           foldToASCII(buffer, length);
-          termAtt.setTermBuffer(output, 0, outputPos);
+          termAtt.copyBuffer(output, 0, outputPos);
           break;
         }
       }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Analyzer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Analyzer.java Sat Apr 10 15:41:27 2010
@@ -84,28 +84,6 @@ public abstract class Analyzer implement
     }
   }
 
-  private static final VirtualMethod<Analyzer> tokenStreamMethod =
-    new VirtualMethod<Analyzer>(Analyzer.class, "tokenStream", String.class, Reader.class);
-  private static final VirtualMethod<Analyzer> reusableTokenStreamMethod =
-    new VirtualMethod<Analyzer>(Analyzer.class, "reusableTokenStream", String.class, Reader.class);
-
-  /** This field contains if the {@link #tokenStream} method was overridden in a
-   * more far away subclass of {@code Analyzer} on the current instance's inheritance path.
-   * If this field is {@code true}, {@link #reusableTokenStream} should delegate to {@link #tokenStream}
-   * instead of using the own implementation.
-   * @deprecated Please declare all implementations of {@link #reusableTokenStream} and {@link #tokenStream}
-   * as {@code final}.
-   */
-  @Deprecated
-  protected final boolean overridesTokenStreamMethod =
-    VirtualMethod.compareImplementationDistance(this.getClass(), tokenStreamMethod, reusableTokenStreamMethod) > 0;
-
-  /** @deprecated This is a no-op since Lucene 3.1. */
-  @Deprecated
-  protected void setOverridesTokenStreamMethod(Class<? extends Analyzer> baseClass) {
-  }
-
-
   /**
    * Invoked before indexing a Fieldable instance if
    * terms have already been added to that field.  This allows custom

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java Sat Apr 10 15:41:27 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 import java.io.Reader;
 
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.CharacterUtils;
 import org.apache.lucene.util.Version;
@@ -78,10 +78,7 @@ public abstract class CharTokenizer exte
   public CharTokenizer(Version matchVersion, Reader input) {
     super(input);
     charUtils = CharacterUtils.getInstance(matchVersion);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    termAtt = addAttribute(TermAttribute.class);
     useOldAPI = useOldAPI(matchVersion);
-    ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
 
   }
   
@@ -99,10 +96,7 @@ public abstract class CharTokenizer exte
       Reader input) {
     super(source, input);
     charUtils = CharacterUtils.getInstance(matchVersion);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    termAtt = addAttribute(TermAttribute.class);
     useOldAPI = useOldAPI(matchVersion);
-    ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
   }
   
   /**
@@ -119,10 +113,7 @@ public abstract class CharTokenizer exte
       Reader input) {
     super(factory, input);
     charUtils = CharacterUtils.getInstance(matchVersion);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    termAtt = addAttribute(TermAttribute.class);
     useOldAPI = useOldAPI(matchVersion);
-    ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
   }
   
   /**
@@ -164,11 +155,11 @@ public abstract class CharTokenizer exte
   private static final int MAX_WORD_LEN = 255;
   private static final int IO_BUFFER_SIZE = 4096;
   
-  private final TermAttribute termAtt;
-  private final OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);;
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
   
   private final CharacterUtils charUtils;
-  private final CharacterBuffer ioBuffer;
+  private final CharacterBuffer ioBuffer = CharacterUtils.newCharacterBuffer(IO_BUFFER_SIZE);
   
   /**
    * @deprecated this will be removed in lucene 4.0
@@ -275,7 +266,7 @@ public abstract class CharTokenizer exte
       return incrementTokenOld();
     int length = 0;
     int start = bufferIndex;
-    char[] buffer = termAtt.termBuffer();
+    char[] buffer = termAtt.buffer();
     while (true) {
       if (bufferIndex >= dataLen) {
         offset += dataLen;
@@ -297,7 +288,7 @@ public abstract class CharTokenizer exte
         if (length == 0)                 // start of token
           start = offset + bufferIndex - 1;
         else if (length >= buffer.length-1) // check if a supplementary could run out of bounds
-          buffer = termAtt.resizeTermBuffer(2+length); // make sure a supplementary fits in the buffer
+          buffer = termAtt.resizeBuffer(2+length); // make sure a supplementary fits in the buffer
         length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
         if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test
           break;
@@ -305,7 +296,7 @@ public abstract class CharTokenizer exte
         break;                           // return 'em
     }
 
-    termAtt.setTermLength(length);
+    termAtt.setLength(length);
     offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
     return true;
     
@@ -320,7 +311,7 @@ public abstract class CharTokenizer exte
   private boolean incrementTokenOld() throws IOException {
     int length = 0;
     int start = bufferIndex;
-    char[] buffer = termAtt.termBuffer();
+    char[] buffer = termAtt.buffer();
     final char[] oldIoBuffer = ioBuffer.getBuffer();
     while (true) {
 
@@ -344,7 +335,7 @@ public abstract class CharTokenizer exte
         if (length == 0)                 // start of token
           start = offset + bufferIndex - 1;
         else if (length == buffer.length)
-          buffer = termAtt.resizeTermBuffer(1+length);
+          buffer = termAtt.resizeBuffer(1+length);
 
         buffer[length++] = normalize(c); // buffer it, normalized
 
@@ -355,7 +346,7 @@ public abstract class CharTokenizer exte
         break;                           // return 'em
     }
 
-    termAtt.setTermLength(length);
+    termAtt.setLength(length);
     offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
     return true;
   }  

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java Sat Apr 10 15:41:27 2010
@@ -1,7 +1,5 @@
 package org.apache.lucene.analysis;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -19,6 +17,8 @@ import org.apache.lucene.analysis.tokena
  * limitations under the License.
  */
 
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
 /**
  * A filter that replaces accented characters in the ISO Latin 1 character set 
  * (ISO-8859-1) by their unaccented equivalent. The case will not be altered.
@@ -35,25 +35,24 @@ import org.apache.lucene.analysis.tokena
 public final class ISOLatin1AccentFilter extends TokenFilter {
   public ISOLatin1AccentFilter(TokenStream input) {
     super(input);
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   private char[] output = new char[256];
   private int outputPos;
-  private TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
     
   @Override
   public final boolean incrementToken() throws java.io.IOException {    
     if (input.incrementToken()) {
-      final char[] buffer = termAtt.termBuffer();
-      final int length = termAtt.termLength();
+      final char[] buffer = termAtt.buffer();
+      final int length = termAtt.length();
       // If no characters actually require rewriting then we
       // just return token as-is:
       for(int i=0;i<length;i++) {
         final char c = buffer[i];
         if (c >= '\u00c0' && c <= '\uFB06') {
           removeAccents(buffer, length);
-          termAtt.setTermBuffer(output, 0, outputPos);
+          termAtt.copyBuffer(output, 0, outputPos);
           break;
         }
       }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordAnalyzer.java Sat Apr 10 15:41:27 2010
@@ -17,36 +17,18 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
-import java.io.IOException;
 import java.io.Reader;
 
 /**
  * "Tokenizes" the entire stream as a single token. This is useful
  * for data like zip codes, ids, and some product names.
  */
-public class KeywordAnalyzer extends Analyzer {
+public final class KeywordAnalyzer extends ReusableAnalyzerBase {
   public KeywordAnalyzer() {
   }
+
   @Override
-  public TokenStream tokenStream(String fieldName,
-                                 final Reader reader) {
-    return new KeywordTokenizer(reader);
-  }
-  @Override
-  public TokenStream reusableTokenStream(String fieldName,
-                                         final Reader reader) throws IOException {
-    if (overridesTokenStreamMethod) {
-      // LUCENE-1678: force fallback to tokenStream() if we
-      // have been subclassed and that subclass overrides
-      // tokenStream but not reusableTokenStream
-      return tokenStream(fieldName, reader);
-    }
-    Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
-    if (tokenizer == null) {
-      tokenizer = new KeywordTokenizer(reader);
-      setPreviousTokenStream(tokenizer);
-    } else
-      tokenizer.reset(reader);
-    return tokenizer;
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    return new TokenStreamComponents(new KeywordTokenizer(reader));
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordMarkerTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordMarkerTokenFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordMarkerTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordMarkerTokenFilter.java Sat Apr 10 15:41:27 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 import java.util.Set;
 
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.Version;
 
 /**
@@ -33,8 +33,8 @@ import org.apache.lucene.util.Version;
  */
 public final class KeywordMarkerTokenFilter extends TokenFilter {
 
-  private final KeywordAttribute keywordAttr;
-  private final TermAttribute termAtt;
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final CharArraySet keywordSet;
 
   /**
@@ -50,8 +50,6 @@ public final class KeywordMarkerTokenFil
   public KeywordMarkerTokenFilter(final TokenStream in,
       final CharArraySet keywordSet) {
     super(in);
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
     this.keywordSet = keywordSet;
   }
 
@@ -73,8 +71,8 @@ public final class KeywordMarkerTokenFil
   @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      keywordAttr.setKeyword(keywordSet.contains(termAtt.termBuffer(), 0,
-          termAtt.termLength()));
+      keywordAttr.setKeyword(keywordSet.contains(termAtt.buffer(), 0,
+          termAtt.length()));
       return true;
     } else
       return false;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordTokenizer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordTokenizer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/KeywordTokenizer.java Sat Apr 10 15:41:27 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 import java.io.Reader;
 
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.AttributeSource;
 
 /**
@@ -31,10 +31,10 @@ public final class KeywordTokenizer exte
   
   private static final int DEFAULT_BUFFER_SIZE = 256;
 
-  private boolean done;
+  private boolean done = false;
   private int finalOffset;
-  private TermAttribute termAtt;
-  private OffsetAttribute offsetAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
   
   public KeywordTokenizer(Reader input) {
     this(input, DEFAULT_BUFFER_SIZE);
@@ -42,24 +42,17 @@ public final class KeywordTokenizer exte
 
   public KeywordTokenizer(Reader input, int bufferSize) {
     super(input);
-    init(bufferSize);
+    termAtt.resizeBuffer(bufferSize);
   }
 
   public KeywordTokenizer(AttributeSource source, Reader input, int bufferSize) {
     super(source, input);
-    init(bufferSize);
+    termAtt.resizeBuffer(bufferSize);
   }
 
   public KeywordTokenizer(AttributeFactory factory, Reader input, int bufferSize) {
     super(factory, input);
-    init(bufferSize);
-  }
-  
-  private void init(int bufferSize) {
-    this.done = false;
-    termAtt = addAttribute(TermAttribute.class);
-    offsetAtt = addAttribute(OffsetAttribute.class);
-    termAtt.resizeTermBuffer(bufferSize);    
+    termAtt.resizeBuffer(bufferSize);
   }
   
   @Override
@@ -68,15 +61,15 @@ public final class KeywordTokenizer exte
       clearAttributes();
       done = true;
       int upto = 0;
-      char[] buffer = termAtt.termBuffer();
+      char[] buffer = termAtt.buffer();
       while (true) {
         final int length = input.read(buffer, upto, buffer.length-upto);
         if (length == -1) break;
         upto += length;
         if (upto == buffer.length)
-          buffer = termAtt.resizeTermBuffer(1+buffer.length);
+          buffer = termAtt.resizeBuffer(1+buffer.length);
       }
-      termAtt.setTermLength(upto);
+      termAtt.setLength(upto);
       finalOffset = correctOffset(upto);
       offsetAtt.setOffset(correctOffset(0), finalOffset);
       return true;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java Sat Apr 10 15:41:27 2010
@@ -19,17 +19,17 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Removes words that are too long or too short from the stream.
  */
 public final class LengthFilter extends TokenFilter {
 
-  final int min;
-  final int max;
+  private final int min;
+  private final int max;
   
-  private TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
 
   /**
    * Build a filter that removes words that are too long or too
@@ -40,7 +40,6 @@ public final class LengthFilter extends 
     super(in);
     this.min = min;
     this.max = max;
-    termAtt = addAttribute(TermAttribute.class);
   }
   
   /**
@@ -50,7 +49,7 @@ public final class LengthFilter extends 
   public final boolean incrementToken() throws IOException {
     // return the first non-stop word found
     while (input.incrementToken()) {
-      int len = termAtt.termLength();
+      int len = termAtt.length();
       if (len >= min && len <= max) {
           return true;
       }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java Sat Apr 10 15:41:27 2010
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.CharacterUtils;
 import org.apache.lucene.util.Version;
 
@@ -34,7 +34,8 @@ import org.apache.lucene.util.Version;
  */
 public final class LowerCaseFilter extends TokenFilter {
   private final CharacterUtils charUtils;
-
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  
   /**
    * Create a new LowerCaseFilter, that normalizes token text to lower case.
    * 
@@ -43,7 +44,6 @@ public final class LowerCaseFilter exten
    */
   public LowerCaseFilter(Version matchVersion, TokenStream in) {
     super(in);
-    termAtt = addAttribute(TermAttribute.class);
     charUtils = CharacterUtils.getInstance(matchVersion);
   }
   
@@ -55,13 +55,11 @@ public final class LowerCaseFilter exten
     this(Version.LUCENE_30, in);
   }
 
-  private TermAttribute termAtt;
-  
   @Override
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      final char[] buffer = termAtt.termBuffer();
-      final int length = termAtt.termLength();
+      final char[] buffer = termAtt.buffer();
+      final int length = termAtt.length();
       for (int i = 0; i < length;) {
        i += Character.toChars(
                Character.toLowerCase(

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java Sat Apr 10 15:41:27 2010
@@ -26,7 +26,6 @@ import org.apache.lucene.document.Numeri
 import org.apache.lucene.search.NumericRangeQuery; // for javadocs
 import org.apache.lucene.search.NumericRangeFilter; // for javadocs
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -118,11 +117,14 @@ public final class NumericTokenStream ex
       this.delegate = delegate;
     }
   
-    @Override
+    @Override @SuppressWarnings("deprecation")
     public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
       if (attClass == NumericTermAttribute.class)
         return new NumericTermAttributeImpl(ts);
-      if (attClass.isAssignableFrom(CharTermAttribute.class) || attClass.isAssignableFrom(TermAttribute.class))
+      if (attClass.isAssignableFrom(CharTermAttribute.class) ||
+          // TODO: remove in 4.0 (deprecated class, also remove the suppress above):
+          attClass.isAssignableFrom(org.apache.lucene.analysis.tokenattributes.TermAttribute.class)
+      )
         throw new IllegalArgumentException("NumericTokenStream does not support CharTermAttribute/TermAttribute.");
       return delegate.createAttributeInstance(attClass);
     }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java Sat Apr 10 15:41:27 2010
@@ -44,7 +44,7 @@ import java.util.HashMap;
  * <p>A PerFieldAnalyzerWrapper can be used like any other analyzer, for both indexing
  * and query parsing.
  */
-public class PerFieldAnalyzerWrapper extends Analyzer {
+public final class PerFieldAnalyzerWrapper extends Analyzer {
   private Analyzer defaultAnalyzer;
   private Map<String,Analyzer> analyzerMap = new HashMap<String,Analyzer>();
 
@@ -99,12 +99,6 @@ public class PerFieldAnalyzerWrapper ext
   
   @Override
   public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    if (overridesTokenStreamMethod) {
-      // LUCENE-1678: force fallback to tokenStream() if we
-      // have been subclassed and that subclass overrides
-      // tokenStream but not reusableTokenStream
-      return tokenStream(fieldName, reader);
-    }
     Analyzer analyzer = analyzerMap.get(fieldName);
     if (analyzer == null)
       analyzer = defaultAnalyzer;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java Sat Apr 10 15:41:27 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.analysis;
 import java.io.IOException;
 
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /** Transforms the token stream as per the Porter stemming algorithm.
     Note: the input to the stemming filter must already be in lower case,
@@ -47,15 +47,12 @@ import org.apache.lucene.analysis.tokena
     </p>
 */
 public final class PorterStemFilter extends TokenFilter {
-  private final PorterStemmer stemmer;
-  private final TermAttribute termAtt;
-  private final KeywordAttribute keywordAttr;
+  private final PorterStemmer stemmer = new PorterStemmer();
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
   public PorterStemFilter(TokenStream in) {
     super(in);
-    stemmer = new PorterStemmer();
-    termAtt = addAttribute(TermAttribute.class);
-    keywordAttr = addAttribute(KeywordAttribute.class);
   }
 
   @Override
@@ -63,8 +60,8 @@ public final class PorterStemFilter exte
     if (!input.incrementToken())
       return false;
 
-    if ((!keywordAttr.isKeyword()) && stemmer.stem(termAtt.termBuffer(), 0, termAtt.termLength()))
-      termAtt.setTermBuffer(stemmer.getResultBuffer(), 0, stemmer.getResultLength());
+    if ((!keywordAttr.isKeyword()) && stemmer.stem(termAtt.buffer(), 0, termAtt.length()))
+      termAtt.copyBuffer(stemmer.getResultBuffer(), 0, stemmer.getResultLength());
     return true;
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java Sat Apr 10 15:41:27 2010
@@ -100,8 +100,8 @@ public abstract class ReusableAnalyzerBa
    * {@link Analyzer#reusableTokenStream(String, Reader)}.
    */
   public static class TokenStreamComponents {
-    final Tokenizer source;
-    final TokenStream sink;
+    protected final Tokenizer source;
+    protected final TokenStream sink;
 
     /**
      * Creates a new {@link TokenStreamComponents} instance.

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/StopFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/StopFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/StopFilter.java Sat Apr 10 15:41:27 2010
@@ -23,7 +23,7 @@ import java.util.Set;
 import java.util.List;
 
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.queryParser.QueryParser; // for javadoc
 import org.apache.lucene.util.Version;
 
@@ -44,8 +44,8 @@ public final class StopFilter extends To
   private final CharArraySet stopWords;
   private boolean enablePositionIncrements = false;
 
-  private TermAttribute termAtt;
-  private PositionIncrementAttribute posIncrAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
 
   /**
    * Construct a token stream filtering the given input.
@@ -104,8 +104,6 @@ public final class StopFilter extends To
     super(input);
     this.stopWords = stopWords instanceof CharArraySet ? (CharArraySet)stopWords : new CharArraySet(matchVersion, stopWords, ignoreCase);  
     this.enablePositionIncrements = enablePositionIncrements;
-    termAtt = addAttribute(TermAttribute.class);
-    posIncrAtt = addAttribute(PositionIncrementAttribute.class);
   }
 
   /**
@@ -257,7 +255,7 @@ public final class StopFilter extends To
     // return the first non-stop word found
     int skippedPositions = 0;
     while (input.incrementToken()) {
-      if (!stopWords.contains(termAtt.termBuffer(), 0, termAtt.termLength())) {
+      if (!stopWords.contains(termAtt.buffer(), 0, termAtt.length())) {
         if (enablePositionIncrements) {
           posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
         }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Sat Apr 10 15:41:27 2010
@@ -42,8 +42,12 @@ import java.util.Set;
  *        are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1068</a>)
  * </ul>
  */
-public class StandardAnalyzer extends Analyzer {
-  private Set<?> stopSet;
+public final class StandardAnalyzer extends StopwordAnalyzerBase {
+
+  /** Default maximum allowed token length */
+  public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
+
+  private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
 
   /**
    * Specifies whether deprecated acronyms should be replaced with HOST type.
@@ -54,7 +58,15 @@ public class StandardAnalyzer extends An
   /** An unmodifiable set containing some common English words that are usually not
   useful for searching. */
   public static final Set<?> STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET; 
-  private final Version matchVersion;
+
+  /** Builds an analyzer with the given stop words.
+   * @param matchVersion Lucene version to match See {@link
+   * <a href="#version">above</a>}
+   * @param stopWords stop words */
+  public StandardAnalyzer(Version matchVersion, Set<?> stopWords) {
+    super(matchVersion, stopWords);
+    replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24);
+  }
 
   /** Builds an analyzer with the default stop words ({@link
    * #STOP_WORDS_SET}).
@@ -65,16 +77,6 @@ public class StandardAnalyzer extends An
     this(matchVersion, STOP_WORDS_SET);
   }
 
-  /** Builds an analyzer with the given stop words.
-   * @param matchVersion Lucene version to match See {@link
-   * <a href="#version">above</a>}
-   * @param stopWords stop words */
-  public StandardAnalyzer(Version matchVersion, Set<?> stopWords) {
-    stopSet = stopWords;
-    replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_24);
-    this.matchVersion = matchVersion;
-  }
-
   /** Builds an analyzer with the stop words from the given file.
    * @see WordlistLoader#getWordSet(File)
    * @param matchVersion Lucene version to match See {@link
@@ -93,28 +95,6 @@ public class StandardAnalyzer extends An
     this(matchVersion, WordlistLoader.getWordSet(stopwords));
   }
 
-  /** Constructs a {@link StandardTokenizer} filtered by a {@link
-  StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. */
-  @Override
-  public TokenStream tokenStream(String fieldName, Reader reader) {
-    StandardTokenizer tokenStream = new StandardTokenizer(matchVersion, reader);
-    tokenStream.setMaxTokenLength(maxTokenLength);
-    TokenStream result = new StandardFilter(tokenStream);
-    result = new LowerCaseFilter(matchVersion, result);
-    result = new StopFilter(matchVersion, result, stopSet);
-    return result;
-  }
-
-  private static final class SavedStreams {
-    StandardTokenizer tokenStream;
-    TokenStream filteredTokenStream;
-  }
-
-  /** Default maximum allowed token length */
-  public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
-
-  private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
-
   /**
    * Set maximum allowed token length.  If a token is seen
    * that exceeds this length then it is discarded.  This
@@ -133,29 +113,19 @@ public class StandardAnalyzer extends An
   }
 
   @Override
-  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
-    if (overridesTokenStreamMethod) {
-      // LUCENE-1678: force fallback to tokenStream() if we
-      // have been subclassed and that subclass overrides
-      // tokenStream but not reusableTokenStream
-      return tokenStream(fieldName, reader);
-    }
-    SavedStreams streams = (SavedStreams) getPreviousTokenStream();
-    if (streams == null) {
-      streams = new SavedStreams();
-      setPreviousTokenStream(streams);
-      streams.tokenStream = new StandardTokenizer(matchVersion, reader);
-      streams.filteredTokenStream = new StandardFilter(streams.tokenStream);
-      streams.filteredTokenStream = new LowerCaseFilter(matchVersion,
-          streams.filteredTokenStream);
-      streams.filteredTokenStream = new StopFilter(matchVersion, streams.filteredTokenStream, stopSet);
-    } else {
-      streams.tokenStream.reset(reader);
-    }
-    streams.tokenStream.setMaxTokenLength(maxTokenLength);
-    
-    streams.tokenStream.setReplaceInvalidAcronym(replaceInvalidAcronym);
-
-    return streams.filteredTokenStream;
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
+    src.setMaxTokenLength(maxTokenLength);
+    src.setReplaceInvalidAcronym(replaceInvalidAcronym);
+    TokenStream tok = new StandardFilter(src);
+    tok = new LowerCaseFilter(matchVersion, tok);
+    tok = new StopFilter(matchVersion, tok, stopwords);
+    return new TokenStreamComponents(src, tok) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
+        return super.reset(reader);
+      }
+    };
   }
 }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java Sat Apr 10 15:41:27 2010
@@ -19,27 +19,24 @@ package org.apache.lucene.analysis.stand
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 /** Normalizes tokens extracted with {@link StandardTokenizer}. */
 
 public final class StandardFilter extends TokenFilter {
 
-
   /** Construct filtering <i>in</i>. */
   public StandardFilter(TokenStream in) {
     super(in);
-    termAtt = addAttribute(TermAttribute.class);
-    typeAtt = addAttribute(TypeAttribute.class);
   }
 
   private static final String APOSTROPHE_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.APOSTROPHE];
   private static final String ACRONYM_TYPE = StandardTokenizer.TOKEN_TYPES[StandardTokenizer.ACRONYM];
 
   // this filters uses attribute type
-  private final TypeAttribute typeAtt;
-  private final TermAttribute termAtt;
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   
   /** Returns the next token in the stream, or null at EOS.
    * <p>Removes <tt>'s</tt> from the end of words.
@@ -51,16 +48,16 @@ public final class StandardFilter extend
       return false;
     }
 
-    char[] buffer = termAtt.termBuffer();
-    final int bufferLength = termAtt.termLength();
+    final char[] buffer = termAtt.buffer();
+    final int bufferLength = termAtt.length();
     final String type = typeAtt.type();
 
     if (type == APOSTROPHE_TYPE &&      // remove 's
-  bufferLength >= 2 &&
+        bufferLength >= 2 &&
         buffer[bufferLength-2] == '\'' &&
         (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
       // Strip last 2 characters off
-      termAtt.setTermLength(bufferLength - 2);
+      termAtt.setLength(bufferLength - 2);
     } else if (type == ACRONYM_TYPE) {      // remove dots
       int upto = 0;
       for(int i=0;i<bufferLength;i++) {
@@ -68,7 +65,7 @@ public final class StandardFilter extend
         if (c != '.')
           buffer[upto++] = c;
       }
-      termAtt.setTermLength(upto);
+      termAtt.setLength(upto);
     }
 
     return true;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/collation/CollationKeyFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/collation/CollationKeyFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/collation/CollationKeyFilter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/collation/CollationKeyFilter.java Sat Apr 10 15:41:27 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.collation;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.IndexableBinaryStringTools;
 
 import java.io.IOException;
@@ -73,8 +73,8 @@ import java.text.Collator;
  * </p>
  */
 public final class CollationKeyFilter extends TokenFilter {
-  private Collator collator = null;
-  private TermAttribute termAtt;
+  private final Collator collator;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
 
   /**
    * @param input Source token stream
@@ -83,23 +83,18 @@ public final class CollationKeyFilter ex
   public CollationKeyFilter(TokenStream input, Collator collator) {
     super(input);
     this.collator = collator;
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      char[] termBuffer = termAtt.termBuffer();
-      String termText = new String(termBuffer, 0, termAtt.termLength());
-      byte[] collationKey = collator.getCollationKey(termText).toByteArray();
+      byte[] collationKey = collator.getCollationKey(termAtt.toString()).toByteArray();
       int encodedLength = IndexableBinaryStringTools.getEncodedLength(
           collationKey, 0, collationKey.length);
-      if (encodedLength > termBuffer.length) {
-        termAtt.resizeTermBuffer(encodedLength);
-      }
-      termAtt.setTermLength(encodedLength);
+      termAtt.resizeBuffer(encodedLength);
+      termAtt.setLength(encodedLength);
       IndexableBinaryStringTools.encode(collationKey, 0, collationKey.length,
-          termAtt.termBuffer(), 0, encodedLength);
+          termAtt.buffer(), 0, encodedLength);
       return true;
     } else {
       return false;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java Sat Apr 10 15:41:27 2010
@@ -17,7 +17,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.DateField;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
@@ -522,7 +522,7 @@ public class QueryParser implements Quer
       source = analyzer.tokenStream(field, new StringReader(queryText));
     }
     CachingTokenFilter buffer = new CachingTokenFilter(source);
-    TermAttribute termAtt = null;
+    CharTermAttribute termAtt = null;
     PositionIncrementAttribute posIncrAtt = null;
     int numTokens = 0;
 
@@ -534,8 +534,8 @@ public class QueryParser implements Quer
       // success==false if we hit an exception
     }
     if (success) {
-      if (buffer.hasAttribute(TermAttribute.class)) {
-        termAtt = buffer.getAttribute(TermAttribute.class);
+      if (buffer.hasAttribute(CharTermAttribute.class)) {
+        termAtt = buffer.getAttribute(CharTermAttribute.class);
       }
       if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
         posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
@@ -581,7 +581,7 @@ public class QueryParser implements Quer
       try {
         boolean hasNext = buffer.incrementToken();
         assert hasNext == true;
-        term = termAtt.term();
+        term = termAtt.toString();
       } catch (IOException e) {
         // safe to ignore, because we know the number of tokens
       }
@@ -596,7 +596,7 @@ public class QueryParser implements Quer
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
             } catch (IOException e) {
               // safe to ignore, because we know the number of tokens
             }
@@ -619,7 +619,7 @@ public class QueryParser implements Quer
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
               if (posIncrAtt != null) {
                 positionIncrement = posIncrAtt.getPositionIncrement();
               }
@@ -659,7 +659,7 @@ public class QueryParser implements Quer
           try {
             boolean hasNext = buffer.incrementToken();
             assert hasNext == true;
-            term = termAtt.term();
+            term = termAtt.toString();
             if (posIncrAtt != null) {
               positionIncrement = posIncrAtt.getPositionIncrement();
             }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj Sat Apr 10 15:41:27 2010
@@ -41,7 +41,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.DateField;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;
@@ -546,7 +546,7 @@ public class QueryParser {
       source = analyzer.tokenStream(field, new StringReader(queryText));
     }
     CachingTokenFilter buffer = new CachingTokenFilter(source);
-    TermAttribute termAtt = null;
+    CharTermAttribute termAtt = null;
     PositionIncrementAttribute posIncrAtt = null;
     int numTokens = 0;
 
@@ -558,8 +558,8 @@ public class QueryParser {
       // success==false if we hit an exception
     }
     if (success) {
-      if (buffer.hasAttribute(TermAttribute.class)) {
-        termAtt = buffer.getAttribute(TermAttribute.class);
+      if (buffer.hasAttribute(CharTermAttribute.class)) {
+        termAtt = buffer.getAttribute(CharTermAttribute.class);
       }
       if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
         posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
@@ -605,7 +605,7 @@ public class QueryParser {
       try {
         boolean hasNext = buffer.incrementToken();
         assert hasNext == true;
-        term = termAtt.term();
+        term = termAtt.toString();
       } catch (IOException e) {
         // safe to ignore, because we know the number of tokens
       }
@@ -620,7 +620,7 @@ public class QueryParser {
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
             } catch (IOException e) {
               // safe to ignore, because we know the number of tokens
             }
@@ -643,7 +643,7 @@ public class QueryParser {
             try {
               boolean hasNext = buffer.incrementToken();
               assert hasNext == true;
-              term = termAtt.term();
+              term = termAtt.toString();
               if (posIncrAtt != null) {
                 positionIncrement = posIncrAtt.getPositionIncrement();
               }
@@ -683,7 +683,7 @@ public class QueryParser {
           try {
             boolean hasNext = buffer.incrementToken();
             assert hasNext == true;
-            term = termAtt.term();
+            term = termAtt.toString();
             if (posIncrAtt != null) {
               positionIncrement = posIncrAtt.getPositionIncrement();
             }

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java Sat Apr 10 15:41:27 2010
@@ -15,7 +15,7 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.DateField;
 import org.apache.lucene.document.DateTools;
 import org.apache.lucene.index.Term;

Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java Sat Apr 10 15:41:27 2010
@@ -28,7 +28,7 @@ import java.util.Map;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.TermFreqVector;
 
 /**
@@ -61,11 +61,11 @@ public class QueryTermVector implements 
           boolean hasMoreTokens = false;
           
           stream.reset(); 
-          TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+          final CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
 
           hasMoreTokens = stream.incrementToken();
           while (hasMoreTokens) {
-            terms.add(termAtt.term());
+            terms.add(termAtt.toString());
             hasMoreTokens = stream.incrementToken();
           }
           processTerms(terms.toArray(new String[terms.size()]));

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sat Apr 10 15:41:27 2010
@@ -83,8 +83,8 @@ public abstract class BaseTokenStreamTes
     assertNotNull(output);
     CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class);
     
-    assertTrue("has no TermAttribute", ts.hasAttribute(TermAttribute.class));
-    TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
+    assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class));
+    CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
     
     OffsetAttribute offsetAtt = null;
     if (startOffsets != null || endOffsets != null || finalOffset != null) {
@@ -108,7 +108,7 @@ public abstract class BaseTokenStreamTes
     for (int i = 0; i < output.length; i++) {
       // extra safety to enforce, that the state is not preserved and also assign bogus values
       ts.clearAttributes();
-      termAtt.setTermBuffer("bogusTerm");
+      termAtt.setEmpty().append("bogusTerm");
       if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243);
       if (typeAtt != null) typeAtt.setType("bogusType");
       if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657);
@@ -117,7 +117,7 @@ public abstract class BaseTokenStreamTes
       assertTrue("token "+i+" does not exist", ts.incrementToken());
       assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled());
       
-      assertEquals("term "+i, output[i], termAtt.term());
+      assertEquals("term "+i, output[i], termAtt.toString());
       if (startOffsets != null)
         assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset());
       if (endOffsets != null)

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java Sat Apr 10 15:41:27 2010
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import java.io.StringReader;
 import java.util.List;
 import java.util.ArrayList;
@@ -33,7 +33,7 @@ public class TestASCIIFoldingFilter exte
       +" ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"));
     ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
 
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
 
     assertTermEquals("Des", filter, termAtt);
     assertTermEquals("mot", filter, termAtt);
@@ -1890,7 +1890,7 @@ public class TestASCIIFoldingFilter exte
 
     TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(inputText.toString()));
     ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     Iterator<String> expectedIter = expectedOutputTokens.iterator();
     while (expectedIter.hasNext()) {
       assertTermEquals(expectedIter.next(), filter, termAtt);
@@ -1898,8 +1898,8 @@ public class TestASCIIFoldingFilter exte
     assertFalse(filter.incrementToken());
   }
   
-  void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt) throws Exception {
+  void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt) throws Exception {
     assertTrue(stream.incrementToken());
-    assertEquals(expected, termAtt.term());
+    assertEquals(expected, termAtt.toString());
   }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestAnalyzers.java Sat Apr 10 15:41:27 2010
@@ -24,7 +24,7 @@ import java.io.Reader;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.index.Payload;
 
 public class TestAnalyzers extends BaseTokenStreamTestCase {
@@ -120,26 +120,6 @@ public class TestAnalyzers extends BaseT
     String[] y = StandardTokenizer.TOKEN_TYPES;
   }
 
-  private static class MyStandardAnalyzer extends StandardAnalyzer {
-    public MyStandardAnalyzer() {
-      super(TEST_VERSION_CURRENT);
-    }
-  
-    @Override
-    public TokenStream tokenStream(String field, Reader reader) {
-      return new WhitespaceAnalyzer(TEST_VERSION_CURRENT).tokenStream(field, reader);
-    }
-  }
-
-  public void testSubclassOverridingOnlyTokenStream() throws Throwable {
-    Analyzer a = new MyStandardAnalyzer();
-    TokenStream ts = a.reusableTokenStream("field", new StringReader("the"));
-    // StandardAnalyzer will discard "the" (it's a
-    // stopword), by my subclass will not:
-    assertTrue(ts.incrementToken());
-    assertFalse(ts.incrementToken());
-  }
-  
   private static class LowerCaseWhitespaceAnalyzer extends Analyzer {
 
     @Override
@@ -202,8 +182,8 @@ public class TestAnalyzers extends BaseT
     String highSurEndingLower = "bogustermboguster\ud801";
     tokenizer.reset(new StringReader(highSurEndingUpper));
     assertTokenStreamContents(filter, new String[] {highSurEndingLower});
-    assertTrue(filter.hasAttribute(TermAttribute.class));
-    char[] termBuffer = filter.getAttribute(TermAttribute.class).termBuffer();
+    assertTrue(filter.hasAttribute(CharTermAttribute.class));
+    char[] termBuffer = filter.getAttribute(CharTermAttribute.class).buffer();
     int length = highSurEndingLower.length();
     assertEquals('\ud801', termBuffer[length - 1]);
     assertEquals('\udc3e', termBuffer[length]);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sat Apr 10 15:41:27 2010
@@ -21,7 +21,7 @@ package org.apache.lucene.analysis;
 import java.io.IOException;
 
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.TermVector;
@@ -43,7 +43,7 @@ public class TestCachingTokenFilter exte
     Document doc = new Document();
     TokenStream stream = new TokenStream() {
       private int index = 0;
-      private TermAttribute termAtt = addAttribute(TermAttribute.class);
+      private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
       private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
       
       @Override
@@ -52,7 +52,7 @@ public class TestCachingTokenFilter exte
           return false;
         } else {
           clearAttributes();
-          termAtt.setTermBuffer(tokens[index++]);
+          termAtt.append(tokens[index++]);
           offsetAtt.setOffset(0,0);
           return true;
         }        
@@ -100,11 +100,10 @@ public class TestCachingTokenFilter exte
   private void checkTokens(TokenStream stream) throws IOException {
     int count = 0;
     
-    TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
-    assertNotNull(termAtt);
+    CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     while (stream.incrementToken()) {
       assertTrue(count < tokens.length);
-      assertEquals(tokens[count], termAtt.term());
+      assertEquals(tokens[count], termAtt.toString());
       count++;
     }
     

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Sat Apr 10 15:41:27 2010
@@ -17,14 +17,14 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import java.io.StringReader;
 
 public class TestISOLatin1AccentFilter extends BaseTokenStreamTestCase {
   public void testU() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"));
     ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     assertTermEquals("Des", filter, termAtt);
     assertTermEquals("mot", filter, termAtt);
     assertTermEquals("cles", filter, termAtt);
@@ -103,8 +103,8 @@ public class TestISOLatin1AccentFilter e
     assertFalse(filter.incrementToken());
   }
   
-  void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt) throws Exception {
+  void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt) throws Exception {
     assertTrue(stream.incrementToken());
-    assertEquals(expected, termAtt.term());
+    assertEquals(expected, termAtt.toString());
   }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestKeywordMarkerTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestKeywordMarkerTokenFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestKeywordMarkerTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestKeywordMarkerTokenFilter.java Sat Apr 10 15:41:27 2010
@@ -6,7 +6,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.junit.Test;
 
 /**
@@ -53,20 +53,20 @@ public class TestKeywordMarkerTokenFilte
 
   public static class LowerCaseFilterMock extends TokenFilter {
 
-    private TermAttribute termAtt;
-    private KeywordAttribute keywordAttr;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
     public LowerCaseFilterMock(TokenStream in) {
       super(in);
-      termAtt = addAttribute(TermAttribute.class);
-      keywordAttr = addAttribute(KeywordAttribute.class);
     }
 
     @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
-        if (!keywordAttr.isKeyword())
-          termAtt.setTermBuffer(termAtt.term().toLowerCase());
+        if (!keywordAttr.isKeyword()) {
+          final String term = termAtt.toString().toLowerCase();
+          termAtt.setEmpty().append(term);
+        }
         return true;
       }
       return false;

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java Sat Apr 10 15:41:27 2010
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis;
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import java.io.StringReader;
 
 public class TestLengthFilter extends BaseTokenStreamTestCase {
@@ -26,14 +26,14 @@ public class TestLengthFilter extends Ba
     TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, 
         new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
     LengthFilter filter = new LengthFilter(stream, 2, 6);
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
 
     assertTrue(filter.incrementToken());
-    assertEquals("short", termAtt.term());
+    assertEquals("short", termAtt.toString());
     assertTrue(filter.incrementToken());
-    assertEquals("ab", termAtt.term());
+    assertEquals("ab", termAtt.toString());
     assertTrue(filter.incrementToken());
-    assertEquals("foo", termAtt.term());
+    assertEquals("foo", termAtt.toString());
     assertFalse(filter.incrementToken());
   }
 

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Sat Apr 10 15:41:27 2010
@@ -2,7 +2,7 @@ package org.apache.lucene.analysis;
 
 import java.io.StringReader;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -30,19 +30,19 @@ public class TestPerFieldAnalzyerWrapper
 
     TokenStream tokenStream = analyzer.tokenStream("field",
                                             new StringReader(text));
-    TermAttribute termAtt = tokenStream.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
 
     assertTrue(tokenStream.incrementToken());
     assertEquals("WhitespaceAnalyzer does not lowercase",
                  "Qwerty",
-                 termAtt.term());
+                 termAtt.toString());
 
     tokenStream = analyzer.tokenStream("special",
                                             new StringReader(text));
-    termAtt = tokenStream.getAttribute(TermAttribute.class);
+    termAtt = tokenStream.getAttribute(CharTermAttribute.class);
     assertTrue(tokenStream.incrementToken());
     assertEquals("SimpleAnalyzer lowercases",
                  "qwerty",
-                 termAtt.term());
+                 termAtt.toString());
   }
 }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Sat Apr 10 15:41:27 2010
@@ -18,7 +18,7 @@ package org.apache.lucene.analysis;
  */
 
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.Version;
 
 import java.io.StringReader;
@@ -51,10 +51,10 @@ public class TestStopAnalyzer extends Ba
     StringReader reader = new StringReader("This is a test of the english stop analyzer");
     TokenStream stream = stop.tokenStream("test", reader);
     assertTrue(stream != null);
-    TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     
     while (stream.incrementToken()) {
-      assertFalse(inValidTokens.contains(termAtt.term()));
+      assertFalse(inValidTokens.contains(termAtt.toString()));
     }
   }
 
@@ -67,11 +67,11 @@ public class TestStopAnalyzer extends Ba
     StringReader reader = new StringReader("This is a good test of the english stop analyzer");
     TokenStream stream = newStop.tokenStream("test", reader);
     assertNotNull(stream);
-    TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
     
     while (stream.incrementToken()) {
-      String text = termAtt.term();
+      String text = termAtt.toString();
       assertFalse(stopWordsSet.contains(text));
       assertEquals(1,posIncrAtt.getPositionIncrement()); // in 2.4 stop tokenizer does not apply increments.
     }
@@ -88,11 +88,11 @@ public class TestStopAnalyzer extends Ba
     TokenStream stream = newStop.tokenStream("test", reader);
     assertNotNull(stream);
     int i = 0;
-    TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
 
     while (stream.incrementToken()) {
-      String text = termAtt.term();
+      String text = termAtt.toString();
       assertFalse(stopWordsSet.contains(text));
       assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
     }

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestStopFilter.java Sat Apr 10 15:41:27 2010
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis;
  */
 
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.English;
 import org.apache.lucene.util.Version;
 
@@ -37,11 +37,11 @@ public class TestStopFilter extends Base
     StringReader reader = new StringReader("Now is The Time");
     Set<String> stopWords = new HashSet<String>(Arrays.asList("is", "the", "Time"));
     TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopWords, false);
-    final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
+    assertEquals("Now", termAtt.toString());
     assertTrue(stream.incrementToken());
-    assertEquals("The", termAtt.term());
+    assertEquals("The", termAtt.toString());
     assertFalse(stream.incrementToken());
   }
 
@@ -49,9 +49,9 @@ public class TestStopFilter extends Base
     StringReader reader = new StringReader("Now is The Time");
     Set<Object> stopWords = new HashSet<Object>(Arrays.asList( "is", "the", "Time" ));
     TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopWords, true);
-    final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
+    assertEquals("Now", termAtt.toString());
     assertFalse(stream.incrementToken());
   }
 
@@ -60,11 +60,11 @@ public class TestStopFilter extends Base
     String[] stopWords = new String[] { "is", "the", "Time" };
     Set<Object> stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
     TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
-    final TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    final CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     assertTrue(stream.incrementToken());
-    assertEquals("Now", termAtt.term());
+    assertEquals("Now", termAtt.toString());
     assertTrue(stream.incrementToken());
-    assertEquals("The", termAtt.term());
+    assertEquals("The", termAtt.toString());
     assertFalse(stream.incrementToken());
   }
 
@@ -117,13 +117,13 @@ public class TestStopFilter extends Base
   private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
     log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
     stpf.setEnablePositionIncrements(enableIcrements);
-    TermAttribute termAtt = stpf.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = stpf.getAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posIncrAtt = stpf.getAttribute(PositionIncrementAttribute.class);
     for (int i=0; i<20; i+=3) {
       assertTrue(stpf.incrementToken());
       log("Token "+i+": "+stpf);
       String w = English.intToEnglish(i).trim();
-      assertEquals("expecting token "+i+" to be "+w,w,termAtt.term());
+      assertEquals("expecting token "+i+" to be "+w,w,termAtt.toString());
       assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,posIncrAtt.getPositionIncrement());
     }
     assertFalse(stpf.incrementToken());

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java Sat Apr 10 15:41:27 2010
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis;
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.English;
 import java.io.IOException;
@@ -59,16 +59,16 @@ public class TestTeeSinkTokenFilter exte
   static final TeeSinkTokenFilter.SinkFilter theFilter = new TeeSinkTokenFilter.SinkFilter() {
     @Override
     public boolean accept(AttributeSource a) {
-      TermAttribute termAtt = a.getAttribute(TermAttribute.class);
-      return termAtt.term().equalsIgnoreCase("The");
+      CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+      return termAtt.toString().equalsIgnoreCase("The");
     }
   };
 
   static final TeeSinkTokenFilter.SinkFilter dogFilter = new TeeSinkTokenFilter.SinkFilter() {
     @Override
     public boolean accept(AttributeSource a) {
-      TermAttribute termAtt = a.getAttribute(TermAttribute.class);
-      return termAtt.term().equalsIgnoreCase("Dogs");
+      CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+      return termAtt.toString().equalsIgnoreCase("Dogs");
     }
   };
 
@@ -135,8 +135,8 @@ public class TestTeeSinkTokenFilter exte
       TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(100));
       teeStream.consumeAllTokens();
       TokenStream stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), 100);
-      TermAttribute tfTok = stream.addAttribute(TermAttribute.class);
-      TermAttribute sinkTok = sink.addAttribute(TermAttribute.class);
+      CharTermAttribute tfTok = stream.addAttribute(CharTermAttribute.class);
+      CharTermAttribute sinkTok = sink.addAttribute(CharTermAttribute.class);
       for (int i=0; stream.incrementToken(); i++) {
         assertTrue(sink.incrementToken());
         assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.equals(sinkTok) == true);

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestToken.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestToken.java Sat Apr 10 15:41:27 2010
@@ -244,8 +244,8 @@ public class TestToken extends LuceneTes
     assertTrue("TypeAttribute is not implemented by SenselessAttributeImpl",
       ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
     
-    assertTrue("TermAttribute is not implemented by Token",
-      ts.addAttribute(TermAttribute.class) instanceof Token);
+    assertTrue("CharTermAttribute is not implemented by Token",
+      ts.addAttribute(CharTermAttribute.class) instanceof Token);
     assertTrue("OffsetAttribute is not implemented by Token",
       ts.addAttribute(OffsetAttribute.class) instanceof Token);
     assertTrue("FlagsAttribute is not implemented by Token",

Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=932749&r1=932748&r2=932749&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Apr 10 15:41:27 2010
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.Whites
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
@@ -152,15 +152,15 @@ public class TestDocumentWriter extends 
               restoreState(state);
               payloadAtt.setPayload(null);
               posIncrAtt.setPositionIncrement(0);
-              termAtt.setTermBuffer(new char[]{'b'}, 0, 1);
+              termAtt.setEmpty().append("b");
               state = null;
               return true;
             }
 
             boolean hasNext = input.incrementToken();
             if (!hasNext) return false;
-            if (Character.isDigit(termAtt.termBuffer()[0])) {
-              posIncrAtt.setPositionIncrement(termAtt.termBuffer()[0] - '0');
+            if (Character.isDigit(termAtt.buffer()[0])) {
+              posIncrAtt.setPositionIncrement(termAtt.buffer()[0] - '0');
             }
             if (first) {
               // set payload on first position only
@@ -174,7 +174,7 @@ public class TestDocumentWriter extends 
 
           }
 
-          TermAttribute termAtt = addAttribute(TermAttribute.class);
+          CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
           PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
           PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);          
         };
@@ -215,7 +215,7 @@ public class TestDocumentWriter extends 
       private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
       private int index = 0;
       
-      private TermAttribute termAtt = addAttribute(TermAttribute.class);
+      private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
       
       @Override
       public boolean incrementToken() throws IOException {
@@ -223,7 +223,7 @@ public class TestDocumentWriter extends 
           return false;
         } else {
           clearAttributes();
-          termAtt.setTermBuffer(tokens[index++]);
+          termAtt.setEmpty().append(tokens[index++]);
           return true;
         }        
       }



Mime
View raw message