lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rm...@apache.org
Subject svn commit: r950008 [3/4] - in /lucene/dev/trunk: lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ lucene/contrib/highlighter/src/test/org/apache/lucene/search...
Date Tue, 01 Jun 2010 10:35:16 GMT
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java Tue Jun  1 10:35:13 2010
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.rever
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.Version;
 
 import java.io.IOException;
@@ -42,7 +42,7 @@ import java.io.IOException;
  */
 public final class ReverseStringFilter extends TokenFilter {
 
-  private TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final char marker;
   private final Version matchVersion;
   private static final char NOMARKER = '\uFFFF';
@@ -131,20 +131,19 @@ public final class ReverseStringFilter e
     super(in);
     this.matchVersion = matchVersion;
     this.marker = marker;
-    termAtt = addAttribute(TermAttribute.class);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
-      int len = termAtt.termLength();
+      int len = termAtt.length();
       if (marker != NOMARKER) {
         len++;
-        termAtt.resizeTermBuffer(len);
-        termAtt.termBuffer()[len - 1] = marker;
+        termAtt.resizeBuffer(len);
+        termAtt.buffer()[len - 1] = marker;
       }
-      reverse( matchVersion, termAtt.termBuffer(), 0, len );
-      termAtt.setTermLength(len);
+      reverse( matchVersion, termAtt.buffer(), 0, len );
+      termAtt.setLength(len);
       return true;
     } else {
       return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Normalizes token text to lower case.
@@ -32,20 +32,19 @@ import org.apache.lucene.analysis.tokena
 @Deprecated
 public final class RussianLowerCaseFilter extends TokenFilter
 {
-    private TermAttribute termAtt;
+    private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    
     public RussianLowerCaseFilter(TokenStream in)
     {
         super(in);
-        termAtt = addAttribute(TermAttribute.class);
     }
 
     @Override
     public final boolean incrementToken() throws IOException
     {
       if (input.incrementToken()) {
-        char[] chArray = termAtt.termBuffer();
-        int chLen = termAtt.termLength();
+        char[] chArray = termAtt.buffer();
+        int chLen = termAtt.length();
         for (int i = 0; i < chLen; i++)
         {
           chArray[i] = Character.toLowerCase(chArray[i]);

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.ru.RussianStemmer;//javadoc @link
 import org.apache.lucene.analysis.snowball.SnowballFilter; // javadoc @link
 
@@ -51,17 +51,14 @@ public final class RussianStemFilter ext
     /**
      * The actual token in the input stream.
      */
-    private RussianStemmer stemmer = null;
+    private RussianStemmer stemmer = new RussianStemmer();
 
-    private final TermAttribute termAtt;
-    private final KeywordAttribute keywordAttr;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
 
     public RussianStemFilter(TokenStream in)
     {
         super(in);
-        stemmer = new RussianStemmer();
-        termAtt = addAttribute(TermAttribute.class);
-        keywordAttr = addAttribute(KeywordAttribute.class);
     }
     /**
      * Returns the next token in the stream, or null at EOS
@@ -71,10 +68,10 @@ public final class RussianStemFilter ext
     {
       if (input.incrementToken()) {
         if(!keywordAttr.isKeyword()) {
-          final String term = termAtt.term();
+          final String term = termAtt.toString();
           final String s = stemmer.stem(term);
           if (s != null && !s.equals(term))
-            termAtt.setTermBuffer(s);
+            termAtt.setEmpty().append(s);
         }
         return true;
       } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Tue Jun  1 10:35:13 2010
@@ -137,10 +137,10 @@ public final class ShingleFilter extends
    */
   private boolean isOutputHere = false;
   
-  private final CharTermAttribute termAtt;
-  private final OffsetAttribute offsetAtt;
-  private final PositionIncrementAttribute posIncrAtt;
-  private final TypeAttribute typeAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
 
 
   /**
@@ -155,10 +155,6 @@ public final class ShingleFilter extends
     super(input);
     setMaxShingleSize(maxShingleSize);
     setMinShingleSize(minShingleSize);
-    this.termAtt = addAttribute(CharTermAttribute.class);
-    this.offsetAtt = addAttribute(OffsetAttribute.class);
-    this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-    this.typeAtt = addAttribute(TypeAttribute.class);
   }
 
   /**

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java Tue Jun  1 10:35:13 2010
@@ -31,11 +31,11 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
 import org.apache.lucene.analysis.payloads.PayloadHelper;
 import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column.Row;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.index.Payload;
 
@@ -193,14 +193,14 @@ public final class ShingleMatrixFilter e
 
   private TokenStream input;
 
-  private TermAttribute termAtt;
+  private CharTermAttribute termAtt;
   private PositionIncrementAttribute posIncrAtt;
   private PayloadAttribute payloadAtt;
   private OffsetAttribute offsetAtt;
   private TypeAttribute typeAtt;
   private FlagsAttribute flagsAtt;
 
-  private TermAttribute in_termAtt;
+  private CharTermAttribute in_termAtt;
   private PositionIncrementAttribute in_posIncrAtt;
   private PayloadAttribute in_payloadAtt;
   private OffsetAttribute in_offsetAtt;
@@ -229,7 +229,7 @@ public final class ShingleMatrixFilter e
     this.ignoringSinglePrefixOrSuffixShingle = ignoringSinglePrefixOrSuffixShingle;
     this.settingsCodec = settingsCodec;
 
-    termAtt = addAttribute(TermAttribute.class);
+    termAtt = addAttribute(CharTermAttribute.class);
     posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     payloadAtt = addAttribute(PayloadAttribute.class);
     offsetAtt = addAttribute(OffsetAttribute.class);
@@ -239,7 +239,7 @@ public final class ShingleMatrixFilter e
     // set the input to be an empty token stream, we already have the data.
     this.input = new EmptyTokenStream();
 
-    in_termAtt = input.addAttribute(TermAttribute.class);
+    in_termAtt = input.addAttribute(CharTermAttribute.class);
     in_posIncrAtt = input.addAttribute(PositionIncrementAttribute.class);
     in_payloadAtt = input.addAttribute(PayloadAttribute.class);
     in_offsetAtt = input.addAttribute(OffsetAttribute.class);
@@ -311,14 +311,14 @@ public final class ShingleMatrixFilter e
     this.spacerCharacter = spacerCharacter;
     this.ignoringSinglePrefixOrSuffixShingle = ignoringSinglePrefixOrSuffixShingle;
     this.settingsCodec = settingsCodec;
-    termAtt = addAttribute(TermAttribute.class);
+    termAtt = addAttribute(CharTermAttribute.class);
     posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     payloadAtt = addAttribute(PayloadAttribute.class);
     offsetAtt = addAttribute(OffsetAttribute.class);
     typeAtt = addAttribute(TypeAttribute.class);
     flagsAtt = addAttribute(FlagsAttribute.class);
 
-    in_termAtt = input.addAttribute(TermAttribute.class);
+    in_termAtt = input.addAttribute(CharTermAttribute.class);
     in_posIncrAtt = input.addAttribute(PositionIncrementAttribute.class);
     in_payloadAtt = input.addAttribute(PayloadAttribute.class);
     in_offsetAtt = input.addAttribute(OffsetAttribute.class);
@@ -377,7 +377,7 @@ public final class ShingleMatrixFilter e
     if (token == null) return false;
 
     clearAttributes();
-    termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
+    termAtt.copyBuffer(token.buffer(), 0, token.length());
     posIncrAtt.setPositionIncrement(token.getPositionIncrement());
     flagsAtt.setFlags(token.getFlags());
     offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -388,7 +388,7 @@ public final class ShingleMatrixFilter e
 
   private Token getNextInputToken(Token token) throws IOException {
     if (!input.incrementToken()) return null;
-    token.setTermBuffer(in_termAtt.termBuffer(), 0, in_termAtt.termLength());
+    token.copyBuffer(in_termAtt.buffer(), 0, in_termAtt.length());
     token.setPositionIncrement(in_posIncrAtt.getPositionIncrement());
     token.setFlags(in_flagsAtt.getFlags());
     token.setOffset(in_offsetAtt.startOffset(), in_offsetAtt.endOffset());
@@ -399,7 +399,7 @@ public final class ShingleMatrixFilter e
 
   private Token getNextToken(Token token) throws IOException {
     if (!this.incrementToken()) return null;
-    token.setTermBuffer(termAtt.termBuffer(), 0, termAtt.termLength());
+    token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
     token.setPositionIncrement(posIncrAtt.getPositionIncrement());
     token.setFlags(flagsAtt.getFlags());
     token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
@@ -441,7 +441,7 @@ public final class ShingleMatrixFilter e
 
         for (int i = 0; i < currentShingleLength; i++) {
           Token shingleToken = currentPermuationTokens.get(i + currentPermutationTokensStartOffset);
-          termLength += shingleToken.termLength();
+          termLength += shingleToken.length();
           shingle.add(shingleToken);
         }
         if (spacerCharacter != null) {
@@ -459,9 +459,9 @@ public final class ShingleMatrixFilter e
           if (spacerCharacter != null && sb.length() > 0) {
             sb.append(spacerCharacter);
           }
-          sb.append(shingleToken.termBuffer(), 0, shingleToken.termLength());
+          sb.append(shingleToken.buffer(), 0, shingleToken.length());
         }
-        reusableToken.setTermBuffer(sb.toString());
+        reusableToken.setEmpty().append(sb);
         updateToken(reusableToken, shingle, currentPermutationTokensStartOffset, currentPermutationRows, currentPermuationTokens);
 
         return reusableToken;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.text.DateFormat;
 import java.text.ParseException;
 import java.util.Date;
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.util.AttributeSource;
 
 /**
@@ -34,7 +34,7 @@ public class DateRecognizerSinkFilter ex
   public static final String DATE_TYPE = "date";
 
   protected DateFormat dateFormat;
-  protected TermAttribute termAtt;
+  protected CharTermAttribute termAtt;
   
   /**
    * Uses {@link java.text.SimpleDateFormat#getDateInstance()} as the {@link java.text.DateFormat} object.
@@ -50,10 +50,10 @@ public class DateRecognizerSinkFilter ex
   @Override
   public boolean accept(AttributeSource source) {
     if (termAtt == null) {
-      termAtt = source.addAttribute(TermAttribute.class);
+      termAtt = source.addAttribute(CharTermAttribute.class);
     }
     try {
-      Date date = dateFormat.parse(termAtt.term());//We don't care about the date, just that we can parse it as a date
+      Date date = dateFormat.parse(termAtt.toString());//We don't care about the date, just that we can parse it as a date
       if (date != null) {
         return true;
       }

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.TokenF
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.LowerCaseFilter;
 import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tr.TurkishLowerCaseFilter; // javadoc @link
 import org.tartarus.snowball.SnowballProgram;
 
@@ -42,7 +42,7 @@ public final class SnowballFilter extend
 
   private final SnowballProgram stemmer;
 
-  private final TermAttribute termAtt = addAttribute(TermAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
   
   public SnowballFilter(TokenStream input, SnowballProgram stemmer) {
@@ -76,16 +76,16 @@ public final class SnowballFilter extend
   public final boolean incrementToken() throws IOException {
     if (input.incrementToken()) {
       if (!keywordAttr.isKeyword()) {
-        char termBuffer[] = termAtt.termBuffer();
-        final int length = termAtt.termLength();
+        char termBuffer[] = termAtt.buffer();
+        final int length = termAtt.length();
         stemmer.setCurrent(termBuffer, length);
         stemmer.stem();
         final char finalTerm[] = stemmer.getCurrentBuffer();
         final int newLength = stemmer.getCurrentBufferLength();
         if (finalTerm != termBuffer)
-          termAtt.setTermBuffer(finalTerm, 0, newLength);
+          termAtt.copyBuffer(finalTerm, 0, newLength);
         else
-          termAtt.setTermLength(newLength);
+          termAtt.setLength(newLength);
       }
       return true;
     } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Tue Jun  1 10:35:13 2010
@@ -133,7 +133,7 @@ public final class SynonymFilter extends
         OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class);
 
         newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset());
-        newTermAtt.copyBuffer(repTok.termBuffer(), 0, repTok.termLength());
+        newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length());
         repPos += repTok.getPositionIncrement();
         if (i==0) repPos=origPos;  // make position of first token equal to original
 

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Tue Jun  1 10:35:13 2010
@@ -103,8 +103,7 @@ public class SynonymMap {
     List<Token> ret = new ArrayList<Token>(strings.size());
     for (String str : strings) {
       //Token newTok = new Token(str,0,0,"SYNONYM");
-      Token newTok = new Token(0,0,"SYNONYM");
-      newTok.setTermBuffer(str.toCharArray(), 0, str.length());
+      Token newTok = new Token(str, 0,0,"SYNONYM");
       ret.add(newTok);
     }
     return ret;
@@ -137,7 +136,7 @@ public class SynonymMap {
     while(tok1!=null || tok2!=null) {
       while (tok1 != null && (pos1 <= pos2 || tok2==null)) {
         Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
-        tok.setTermBuffer(tok1.termBuffer(), 0, tok1.termLength());
+        tok.copyBuffer(tok1.buffer(), 0, tok1.length());
         tok.setPositionIncrement(pos1-pos);
         result.add(tok);
         pos=pos1;
@@ -146,7 +145,7 @@ public class SynonymMap {
       }
       while (tok2 != null && (pos2 <= pos1 || tok1==null)) {
         Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
-        tok.setTermBuffer(tok2.termBuffer(), 0, tok2.termLength());
+        tok.copyBuffer(tok2.buffer(), 0, tok2.length());
         tok.setPositionIncrement(pos2-pos);
         result.add(tok);
         pos=pos2;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
 
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 /**
  * Normalizes Turkish token text to lower case.
@@ -37,7 +37,7 @@ public final class TurkishLowerCaseFilte
   private static final int LATIN_SMALL_LETTER_I = '\u0069';
   private static final int LATIN_SMALL_LETTER_DOTLESS_I = '\u0131';
   private static final int COMBINING_DOT_ABOVE = '\u0307';
-  private final TermAttribute termAtt;
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
   
   /**
    * Create a new TurkishLowerCaseFilter, that normalizes Turkish token text 
@@ -47,7 +47,6 @@ public final class TurkishLowerCaseFilte
    */
   public TurkishLowerCaseFilter(TokenStream in) {
     super(in);
-    termAtt = addAttribute(TermAttribute.class);
   }
   
   @Override
@@ -55,8 +54,8 @@ public final class TurkishLowerCaseFilte
     boolean iOrAfter = false;
     
     if (input.incrementToken()) {
-      final char[] buffer = termAtt.termBuffer();
-      int length = termAtt.termLength();
+      final char[] buffer = termAtt.buffer();
+      int length = termAtt.length();
       for (int i = 0; i < length;) {
         final int ch = Character.codePointAt(buffer, i);
     
@@ -88,7 +87,7 @@ public final class TurkishLowerCaseFilte
         i += Character.toChars(Character.toLowerCase(ch), buffer, i);
       }
       
-      termAtt.setTermLength(length);
+      termAtt.setLength(length);
       return true;
     } else
       return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java Tue Jun  1 10:35:13 2010
@@ -18,10 +18,10 @@
 package org.apache.lucene.analysis.wikipedia;
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
 
@@ -116,11 +116,11 @@ public final class WikipediaTokenizer ex
   private Set<String> untokenizedTypes = Collections.emptySet();
   private Iterator<AttributeSource.State> tokens = null;
   
-  private OffsetAttribute offsetAtt;
-  private TypeAttribute typeAtt;
-  private PositionIncrementAttribute posIncrAtt;
-  private TermAttribute termAtt;
-  private FlagsAttribute flagsAtt;
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
 
   /**
    * Creates a new instance of the {@link WikipediaTokenizer}. Attaches the
@@ -176,12 +176,7 @@ public final class WikipediaTokenizer ex
   
   private void init(int tokenOutput, Set<String> untokenizedTypes) {
     this.tokenOutput = tokenOutput;
-    this.untokenizedTypes = untokenizedTypes;
-    this.offsetAtt = addAttribute(OffsetAttribute.class);
-    this.typeAtt = addAttribute(TypeAttribute.class);
-    this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-    this.termAtt = addAttribute(TermAttribute.class);
-    this.flagsAtt = addAttribute(FlagsAttribute.class);    
+    this.untokenizedTypes = untokenizedTypes;    
   }
   
   /*
@@ -245,8 +240,9 @@ public final class WikipediaTokenizer ex
       lastPos = currPos + numAdded;
     }
     //trim the buffer
+    // TODO: this is inefficient
     String s = buffer.toString().trim();
-    termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
+    termAtt.setEmpty().append(s);
     offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
     flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
     //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
@@ -283,8 +279,9 @@ public final class WikipediaTokenizer ex
       lastPos = currPos + numAdded;
     }
     //trim the buffer
+    // TODO: this is inefficient
     String s = buffer.toString().trim();
-    termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
+    termAtt.setEmpty().append(s);
     offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
     flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
     //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
@@ -298,7 +295,7 @@ public final class WikipediaTokenizer ex
   private void setupToken() {
     scanner.getText(termAtt);
     final int start = scanner.yychar();
-    offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.termLength()));
+    offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
   }
 
   /*

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Tue Jun  1 10:35:13 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 5/31/10 3:11 PM */
 
 package org.apache.lucene.analysis.wikipedia;
 
@@ -19,14 +19,14 @@ package org.apache.lucene.analysis.wikip
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 17.05.10 14:51 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * on 5/31/10 3:11 PM from the specification file
+ * <tt>C:/Users/rmuir/workspace/solrcene/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
  */
 class WikipediaTokenizerImpl {
 
@@ -37,16 +37,16 @@ class WikipediaTokenizerImpl {
   private static final int ZZ_BUFFERSIZE = 16384;
 
   /** lexical states */
-  public static final int CATEGORY_STATE = 2;
-  public static final int DOUBLE_EQUALS_STATE = 14;
+  public static final int THREE_SINGLE_QUOTES_STATE = 10;
   public static final int EXTERNAL_LINK_STATE = 6;
+  public static final int DOUBLE_EQUALS_STATE = 14;
   public static final int INTERNAL_LINK_STATE = 4;
   public static final int DOUBLE_BRACE_STATE = 16;
-  public static final int FIVE_SINGLE_QUOTES_STATE = 12;
+  public static final int CATEGORY_STATE = 2;
+  public static final int YYINITIAL = 0;
   public static final int STRING = 18;
+  public static final int FIVE_SINGLE_QUOTES_STATE = 12;
   public static final int TWO_SINGLE_QUOTES_STATE = 8;
-  public static final int YYINITIAL = 0;
-  public static final int THREE_SINGLE_QUOTES_STATE = 10;
 
   /**
    * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
@@ -487,8 +487,8 @@ public final int getPositionIncrement(){
 /**
  * Fills Lucene token with the current token text.
  */
-final void getText(TermAttribute t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+final void getText(CharTermAttribute t) {
+  t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
 final int setText(StringBuilder buffer){
@@ -803,184 +803,184 @@ final int setText(StringBuilder buffer){
       zzMarkedPos = zzMarkedPosL;
 
       switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
-        case 25: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
+        case 16: 
+          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 46: break;
-        case 30: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
+        case 39: 
+          { positionInc = 1; return ACRONYM;
           }
         case 47: break;
-        case 41: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
+        case 8: 
+          { /* ignore */
           }
         case 48: break;
-        case 14: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
+        case 20: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
           }
         case 49: break;
-        case 23: 
-          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
+        case 35: 
+          { positionInc = 1; return COMPANY;
           }
         case 50: break;
-        case 34: 
-          { positionInc = 1; return NUM;
+        case 4: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
           }
         case 51: break;
-        case 18: 
-          { /* ignore STRING */
+        case 25: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);
           }
         case 52: break;
-        case 12: 
-          { currentTokType = ITALICS; numWikiTokensSeen++;  yybegin(STRING); return currentTokType;/*italics*/
+        case 43: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
           }
         case 53: break;
-        case 37: 
-          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
+        case 22: 
+          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
           }
         case 54: break;
-        case 31: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
+        case 34: 
+          { positionInc = 1; return NUM;
           }
         case 55: break;
-        case 10: 
-          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
+        case 32: 
+          { positionInc = 1; return APOSTROPHE;
           }
         case 56: break;
-        case 38: 
-          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
+        case 23: 
+          { numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);
           }
         case 57: break;
-        case 19: 
-          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
+        case 21: 
+          { yybegin(STRING); return currentTokType;/*pipe*/
           }
         case 58: break;
-        case 11: 
-          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE);
+        case 2: 
+          { positionInc = 1; return ALPHANUM;
           }
         case 59: break;
-        case 1: 
-          { numWikiTokensSeen = 0;  positionInc = 1;
+        case 29: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE);
           }
         case 60: break;
-        case 33: 
-          { positionInc = 1; return HOST;
+        case 17: 
+          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
           }
         case 61: break;
-        case 3: 
-          { positionInc = 1; return CJ;
+        case 44: 
+          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
           }
         case 62: break;
-        case 17: 
-          { yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;
+        case 26: 
+          { yybegin(YYINITIAL);
           }
         case 63: break;
-        case 32: 
-          { positionInc = 1; return APOSTROPHE;
+        case 3: 
+          { positionInc = 1; return CJ;
           }
         case 64: break;
-        case 8: 
-          { /* ignore */
+        case 38: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/
           }
         case 65: break;
-        case 4: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);
+        case 15: 
+          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
           }
         case 66: break;
-        case 2: 
-          { positionInc = 1; return ALPHANUM;
+        case 30: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/
           }
         case 67: break;
-        case 26: 
-          { yybegin(YYINITIAL);
+        case 6: 
+          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 68: break;
-        case 43: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);
+        case 5: 
+          { positionInc = 1;
           }
         case 69: break;
-        case 36: 
-          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE);
+        case 19: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/
           }
         case 70: break;
-        case 13: 
-          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
+        case 42: 
+          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
           }
         case 71: break;
-        case 24: 
-          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
-          }
-        case 72: break;
         case 27: 
           { numLinkToks = 0; yybegin(YYINITIAL);
           }
+        case 72: break;
+        case 11: 
+          { currentTokType = BOLD;  yybegin(THREE_SINGLE_QUOTES_STATE);
+          }
         case 73: break;
-        case 15: 
-          { currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);
+        case 13: 
+          { currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);
           }
         case 74: break;
-        case 28: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
+        case 14: 
+          { yybegin(STRING); numWikiTokensSeen++; return currentTokType;
           }
         case 75: break;
-        case 39: 
-          { positionInc = 1; return ACRONYM;
+        case 45: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
           }
         case 76: break;
-        case 29: 
-          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0;  yybegin(INTERNAL_LINK_STATE);
+        case 28: 
+          { currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);
           }
         case 77: break;
-        case 7: 
-          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
+        case 37: 
+          { numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/
           }
         case 78: break;
-        case 16: 
-          { currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;
+        case 9: 
+          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
           }
         case 79: break;
-        case 20: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);
+        case 7: 
+          { yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;
           }
         case 80: break;
-        case 35: 
-          { positionInc = 1; return COMPANY;
+        case 24: 
+          { numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);
           }
         case 81: break;
         case 40: 
           { positionInc = 1; return EMAIL;
           }
         case 82: break;
-        case 42: 
-          { positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;
+        case 1: 
+          { numWikiTokensSeen = 0;  positionInc = 1;
           }
         case 83: break;
-        case 6: 
-          { yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;
+        case 18: 
+          { /* ignore STRING */
           }
         case 84: break;
-        case 44: 
-          { currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);
+        case 36: 
+          { currentTokType = BOLD_ITALICS;  yybegin(FIVE_SINGLE_QUOTES_STATE);
           }
         case 85: break;
-        case 5: 
-          { positionInc = 1;
+        case 33: 
+          { positionInc = 1; return HOST;
           }
         case 86: break;
-        case 9: 
-          { if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;
+        case 31: 
+          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);
           }
         case 87: break;
-        case 45: 
-          { numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);
+        case 41: 
+          { numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/
           }
         case 88: break;
-        case 22: 
-          { numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}
+        case 12: 
+          { currentTokType = ITALICS; numWikiTokensSeen++;  yybegin(STRING); return currentTokType;/*italics*/
           }
         case 89: break;
-        case 21: 
-          { yybegin(STRING); return currentTokType;/*pipe*/
+        case 10: 
+          { numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);
           }
         case 90: break;
         default: 

Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Tue Jun  1 10:35:13 2010
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.wikip
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 %%
 
@@ -81,8 +81,8 @@ public final int getPositionIncrement(){
 /**
  * Fills Lucene token with the current token text.
  */
-final void getText(TermAttribute t) {
-  t.setTermBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+final void getText(CharTermAttribute t) {
+  t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 }
 
 final int setText(StringBuilder buffer){

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.compo
  * limitations under the License.
  */
 
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
@@ -27,7 +25,7 @@ import org.apache.lucene.analysis.BaseTo
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {
   public void testHyphenationCompoundWordsDA() throws Exception {
@@ -176,15 +174,15 @@ public class TestCompoundWordTokenFilter
         CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
         CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false);
     
-    TermAttribute termAtt = tf.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = tf.getAttribute(CharTermAttribute.class);
     assertTrue(tf.incrementToken());
-    assertEquals("Rindfleischüberwachungsgesetz", termAtt.term());
+    assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
     assertTrue(tf.incrementToken());
-    assertEquals("Rind", termAtt.term());
+    assertEquals("Rind", termAtt.toString());
     wsTokenizer.reset(new StringReader("Rindfleischüberwachungsgesetz"));
     tf.reset();
     assertTrue(tf.incrementToken());
-    assertEquals("Rindfleischüberwachungsgesetz", termAtt.term());
+    assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
   }
 
   private Reader getHyphenationReader() throws Exception {

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java Tue Jun  1 10:35:13 2010
@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.BaseTo
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
@@ -50,9 +51,9 @@ public class TestElision extends BaseTok
 
   private List<String> filter(TokenFilter filter) throws IOException {
     List<String> tas = new ArrayList<String>();
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     while (filter.incrementToken()) {
-      tas.add(termAtt.term());
+      tas.add(termAtt.toString());
     }
     return tas;
   }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -41,8 +41,6 @@ public class TestPrefixAndSuffixAwareTok
 
   private static Token createToken(String term, int start, int offset)
   {
-    Token token = new Token(start, offset);
-    token.setTermBuffer(term);
-    return token;
+    return new Token(term, start, offset);
   }
 }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -52,8 +52,6 @@ public class TestPrefixAwareTokenFilter 
 
   private static Token createToken(String term, int start, int offset)
   {
-    Token token = new Token(start, offset);
-    token.setTermBuffer(term);
-    return token;
+    return new Token(term, start, offset);
   }
 }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -51,7 +51,7 @@ public class TestRemoveDuplicatesTokenFi
             if (toks.hasNext()) {
               clearAttributes();
               Token tok = toks.next();
-              termAtt.setEmpty().append(tok.term());
+              termAtt.setEmpty().append(tok);
               offsetAtt.setOffset(tok.startOffset(), tok.endOffset());
               posIncAtt.setPositionIncrement(tok.getPositionIncrement());
               return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java Tue Jun  1 10:35:13 2010
@@ -22,14 +22,14 @@ import java.io.IOException;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 public class TestSingleTokenTokenFilter extends LuceneTestCase {
 
   public void test() throws IOException {
     Token token = new Token();
     SingleTokenTokenStream ts = new SingleTokenTokenStream(token);
-    AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(TermAttribute.class);
+    AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(CharTermAttribute.class);
     assertTrue(tokenAtt instanceof Token);
     ts.reset();
 

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java Tue Jun  1 10:35:13 2010
@@ -97,7 +97,7 @@ public class TestTrimFilter extends Base
       else {
         clearAttributes();
         Token token = tokens[index++];
-        termAtt.setEmpty().append(token.term());
+        termAtt.setEmpty().append(token);
         offsetAtt.setOffset(token.startOffset(), token.endOffset());
         posIncAtt.setPositionIncrement(token.getPositionIncrement());
         flagsAtt.setFlags(token.getFlags());

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java Tue Jun  1 10:35:13 2010
@@ -18,8 +18,8 @@ package org.apache.lucene.analysis.paylo
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.Payload;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -32,7 +32,7 @@ public class DelimitedPayloadTokenFilter
     DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
       (new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)), 
        DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
     assertTermEquals("The", filter, termAtt, payAtt, null);
     assertTermEquals("quick", filter, termAtt, payAtt, "JJ".getBytes("UTF-8"));
@@ -70,7 +70,7 @@ public class DelimitedPayloadTokenFilter
   public void testFloatEncoding() throws Exception {
     String test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
     DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)), '|', new FloatEncoder());
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
     assertTermEquals("The", filter, termAtt, payAtt, null);
     assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeFloat(1.0f));
@@ -88,7 +88,7 @@ public class DelimitedPayloadTokenFilter
   public void testIntEncoding() throws Exception {
     String test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
     DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test)), '|', new IntegerEncoder());
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
     assertTermEquals("The", filter, termAtt, payAtt, null);
     assertTermEquals("quick", filter, termAtt, payAtt, PayloadHelper.encodeInt(1));
@@ -104,10 +104,10 @@ public class DelimitedPayloadTokenFilter
   }
 
   void assertTermEquals(String expected, TokenStream stream, byte[] expectPay) throws Exception {
-    TermAttribute termAtt = stream.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
     PayloadAttribute payloadAtt = stream.getAttribute(PayloadAttribute.class);
     assertTrue(stream.incrementToken());
-    assertEquals(expected, termAtt.term());
+    assertEquals(expected, termAtt.toString());
     Payload payload = payloadAtt.getPayload();
     if (payload != null) {
       assertTrue(payload.length() + " does not equal: " + expectPay.length, payload.length() == expectPay.length);
@@ -121,9 +121,9 @@ public class DelimitedPayloadTokenFilter
   }
 
 
-  void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt, PayloadAttribute payAtt, byte[] expectPay) throws Exception {
+  void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt, PayloadAttribute payAtt, byte[] expectPay) throws Exception {
     assertTrue(stream.incrementToken());
-    assertEquals(expected, termAtt.term());
+    assertEquals(expected, termAtt.toString());
     Payload payload = payAtt.getPayload();
     if (payload != null) {
       assertTrue(payload.length() + " does not equal: " + expectPay.length, payload.length() == expectPay.length);

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java Tue Jun  1 10:35:13 2010
@@ -20,8 +20,8 @@ import org.apache.lucene.analysis.BaseTo
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 import java.io.IOException;
@@ -39,11 +39,11 @@ public class NumericPayloadTokenFilterTe
 
     NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test))), 3, "D");
     boolean seenDogs = false;
-    TermAttribute termAtt = nptf.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
     TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
     PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
     while (nptf.incrementToken()) {
-      if (termAtt.term().equals("dogs")) {
+      if (termAtt.toString().equals("dogs")) {
         seenDogs = true;
         assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
         assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
@@ -60,19 +60,17 @@ public class NumericPayloadTokenFilterTe
   }
 
   private final class WordTokenFilter extends TokenFilter {
-    private TermAttribute termAtt;
-    private TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
     
     private WordTokenFilter(TokenStream input) {
       super(input);
-      termAtt = addAttribute(TermAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
     }
     
     @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
-        if (termAtt.term().equals("dogs"))
+        if (termAtt.toString().equals("dogs"))
           typeAtt.setType("D");
         return true;
       } else {

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java Tue Jun  1 10:35:13 2010
@@ -21,7 +21,7 @@ import org.apache.lucene.analysis.TokenF
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 import java.io.IOException;
@@ -39,12 +39,12 @@ public class TypeAsPayloadTokenFilterTes
 
     TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(test))));
     int count = 0;
-    TermAttribute termAtt = nptf.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
     TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
     PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
     
     while (nptf.incrementToken()) {
-      assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0]))));
+      assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.buffer()[0]))));
       assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
       String type = new String(payloadAtt.getPayload().getData(), "UTF-8");
       assertTrue(type + " is not equal to " + typeAtt.type(), type.equals(typeAtt.type()) == true);
@@ -55,19 +55,17 @@ public class TypeAsPayloadTokenFilterTes
   }
 
   private final class WordTokenFilter extends TokenFilter {
-    private TermAttribute termAtt;
-    private TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
     
     private WordTokenFilter(TokenStream input) {
       super(input);
-      termAtt = addAttribute(TermAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
     }
 
     @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
-        typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0])));
+        typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.buffer()[0])));
         return true;
       } else {
         return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java Tue Jun  1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.shingle.ShingleFilter;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 
 public class PositionFilterTest extends BaseTokenStreamTestCase {
 
@@ -30,19 +30,18 @@ public class PositionFilterTest extends 
 
     protected int index = 0;
     protected String[] testToken;
-    protected TermAttribute termAtt;
+    protected final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
 
     public TestTokenStream(String[] testToken) {
       super();
       this.testToken = testToken;
-      termAtt = addAttribute(TermAttribute.class);
     }
 
     @Override
     public final boolean incrementToken() throws IOException {
       clearAttributes();
       if (index < testToken.length) {
-        termAtt.setTermBuffer(testToken[index++]);
+        termAtt.setEmpty().append(testToken[index++]);
         return true;
       } else {
         return false;

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Tue Jun  1 10:35:13 2010
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.core.LetterTokenizer;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexReader;
@@ -176,9 +175,6 @@ public class QueryAutoStopWordAnalyzerTe
     QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
     a.addStopWords(reader, 10);
     TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
-    TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
-    assertTrue(ts.incrementToken());
-    assertEquals("this", termAtt.term());
-    assertFalse(ts.incrementToken());
+    assertTokenStreamContents(ts, new String[] { "this" });
   }
 }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java Tue Jun  1 10:35:13 2010
@@ -21,46 +21,22 @@ import java.io.StringReader;
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.util.Version;
 
 public class TestReverseStringFilter extends BaseTokenStreamTestCase {
   public void testFilter() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, 
         new StringReader("Do have a nice day"));     // 1-4 length string
     ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream);
-    TermAttribute text = filter.getAttribute(TermAttribute.class);
-    assertTrue(filter.incrementToken());
-    assertEquals("oD", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("evah", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("a", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("ecin", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("yad", text.term());
-    assertFalse(filter.incrementToken());
+    assertTokenStreamContents(filter, new String[] { "oD", "evah", "a", "ecin", "yad" });
   }
   
   public void testFilterWithMark() throws Exception {
     TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(
         "Do have a nice day")); // 1-4 length string
     ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream, '\u0001');
-    TermAttribute text = filter
-        .getAttribute(TermAttribute.class);
-    assertTrue(filter.incrementToken());
-    assertEquals("\u0001oD", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("\u0001evah", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("\u0001a", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("\u0001ecin", text.term());
-    assertTrue(filter.incrementToken());
-    assertEquals("\u0001yad", text.term());
-    assertFalse(filter.incrementToken());
+    assertTokenStreamContents(filter, 
+        new String[] { "\u0001oD", "\u0001evah", "\u0001a", "\u0001ecin", "\u0001yad" });
   }
 
   public void testReverseString() throws Exception {

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java Tue Jun  1 10:35:13 2010
@@ -17,17 +17,13 @@ package org.apache.lucene.analysis.ru;
  * limitations under the License.
  */
 
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.StringReader;
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
 import org.apache.lucene.util.Version;
 
@@ -65,8 +61,8 @@ public class TestRussianAnalyzer extends
             new RussianLetterTokenizer(TEST_VERSION_CURRENT,
                 sampleUnicode);
 
-        TermAttribute text = in.getAttribute(TermAttribute.class);
-        TermAttribute sampleText = sample.getAttribute(TermAttribute.class);
+        CharTermAttribute text = in.getAttribute(CharTermAttribute.class);
+        CharTermAttribute sampleText = sample.getAttribute(CharTermAttribute.class);
 
         for (;;)
         {
@@ -76,34 +72,21 @@ public class TestRussianAnalyzer extends
             boolean nextSampleToken = sample.incrementToken();
             assertEquals(
                 "Unicode",
-                text.term(),
+                text.toString(),
                 nextSampleToken == false
                 ? null
-                : sampleText.term());
+                : sampleText.toString());
         }
 
         inWords.close();
         sampleUnicode.close();
     }
     
-    public void testDigitsInRussianCharset() 
+    /** Check that RussianAnalyzer doesnt discard any numbers */
+    public void testDigitsInRussianCharset() throws IOException
     {
-        Reader reader = new StringReader("text 1000");
-        RussianAnalyzer ra = new RussianAnalyzer(TEST_VERSION_CURRENT);
-        TokenStream stream = ra.tokenStream("", reader);
-
-        TermAttribute termText = stream.getAttribute(TermAttribute.class);
-        try {
-            assertTrue(stream.incrementToken());
-            assertEquals("text", termText.term());
-            assertTrue(stream.incrementToken());
-            assertEquals("RussianAnalyzer's tokenizer skips numbers from input text", "1000", termText.term());
-            assertFalse(stream.incrementToken());
-        }
-        catch (IOException e)
-        {
-            fail("unexpected IOException");
-        }
+      RussianAnalyzer ra = new RussianAnalyzer(TEST_VERSION_CURRENT);
+      assertAnalyzesTo(ra, "text 1000", new String[] { "text", "1000" });
     }
     
     /** @deprecated remove this test in Lucene 4.0: stopwords changed */

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Tue Jun  1 10:35:13 2010
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.TokenS
 import org.apache.lucene.analysis.core.LetterTokenizer;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -159,11 +159,11 @@ public class ShingleAnalyzerWrapperTest 
     int j = -1;
     
     PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     
     while (ts.incrementToken()) {
       j += posIncrAtt.getPositionIncrement();
-      String termText = termAtt.term();
+      String termText = termAtt.toString();
       q.add(new Term("content", termText), j);
     }
 
@@ -186,10 +186,10 @@ public class ShingleAnalyzerWrapperTest 
     TokenStream ts = analyzer.tokenStream("content",
                                           new StringReader("test sentence"));
     
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     
     while (ts.incrementToken()) {
-      String termText =  termAtt.term();
+      String termText =  termAtt.toString();
       q.add(new TermQuery(new Term("content", termText)),
             BooleanClause.Occur.SHOULD);
     }

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java Tue Jun  1 10:35:13 2010
@@ -31,7 +31,12 @@ import org.apache.lucene.analysis.miscel
 import org.apache.lucene.analysis.payloads.PayloadHelper;
 import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix;
 import org.apache.lucene.analysis.shingle.ShingleMatrixFilter.Matrix.Column;
-import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
 
@@ -415,7 +420,7 @@ public class TestShingleMatrixFilter ext
 
   private Token tokenFactory(String text, int posIncr, int startOffset, int endOffset) {
     Token token = new Token(startOffset, endOffset);
-    token.setTermBuffer(text);
+    token.setEmpty().append(text);
     token.setPositionIncrement(posIncr);
     return token;
   }
@@ -427,7 +432,7 @@ public class TestShingleMatrixFilter ext
 
   private Token tokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset) {
     Token token = new Token(startOffset, endOffset);
-    token.setTermBuffer(text);
+    token.setEmpty().append(text);
     token.setPositionIncrement(posIncr);
     ShingleMatrixFilter.defaultSettingsCodec.setWeight(token, weight);
     return token;
@@ -435,7 +440,7 @@ public class TestShingleMatrixFilter ext
 
   private Token tokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset, ShingleMatrixFilter.TokenPositioner positioner) {
     Token token = new Token(startOffset, endOffset);
-    token.setTermBuffer(text);
+    token.setEmpty().append(text);
     token.setPositionIncrement(posIncr);
     ShingleMatrixFilter.defaultSettingsCodec.setWeight(token, weight);
     ShingleMatrixFilter.defaultSettingsCodec.setTokenPositioner(token, positioner);
@@ -445,20 +450,20 @@ public class TestShingleMatrixFilter ext
   // assert-methods start here
 
   private void assertNext(TokenStream ts, String text) throws IOException {
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
 
     assertTrue(ts.incrementToken());
-    assertEquals(text, termAtt.term());
+    assertEquals(text, termAtt.toString());
   }
 
   private void assertNext(TokenStream ts, String text, int positionIncrement, float boost, int startOffset, int endOffset) throws IOException {
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
     PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
     OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
     
     assertTrue(ts.incrementToken());
-    assertEquals(text, termAtt.term());
+    assertEquals(text, termAtt.toString());
     assertEquals(positionIncrement, posIncrAtt.getPositionIncrement());
     assertEquals(boost, payloadAtt.getPayload() == null ? 1f : PayloadHelper.decodeFloat(payloadAtt.getPayload().getData()), 0);
     assertEquals(startOffset, offsetAtt.startOffset());
@@ -466,11 +471,11 @@ public class TestShingleMatrixFilter ext
   }
   
   private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException {
-    TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
     OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
 
     assertTrue(ts.incrementToken());
-    assertEquals(text, termAtt.term());
+    assertEquals(text, termAtt.toString());
     assertEquals(startOffset, offsetAtt.startOffset());
     assertEquals(endOffset, offsetAtt.endOffset());
   }
@@ -478,7 +483,7 @@ public class TestShingleMatrixFilter ext
   private static Token createToken(String term, int start, int offset)
   {
     Token token = new Token(start, offset);
-    token.setTermBuffer(term);
+    token.setEmpty().append(term);
     return token;
   }
 
@@ -486,21 +491,15 @@ public class TestShingleMatrixFilter ext
   public final static class TokenListStream extends TokenStream {
 
     private Collection<Token> tokens;
-    TermAttribute termAtt;
-    PositionIncrementAttribute posIncrAtt;
-    PayloadAttribute payloadAtt;
-    OffsetAttribute offsetAtt;
-    TypeAttribute typeAtt;
-    FlagsAttribute flagsAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+    private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+    private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
     
     public TokenListStream(Collection<Token> tokens) {
       this.tokens = tokens;
-      termAtt = addAttribute(TermAttribute.class);
-      posIncrAtt = addAttribute(PositionIncrementAttribute.class);
-      payloadAtt = addAttribute(PayloadAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
-      flagsAtt = addAttribute(FlagsAttribute.class);
     }
 
     private Iterator<Token> iterator;
@@ -515,7 +514,7 @@ public class TestShingleMatrixFilter ext
       }
       Token prototype = iterator.next();
       clearAttributes();
-      termAtt.setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
+      termAtt.copyBuffer(prototype.buffer(), 0, prototype.length());
       posIncrAtt.setPositionIncrement(prototype.getPositionIncrement());
       flagsAtt.setFlags(prototype.getFlags());
       offsetAtt.setOffset(prototype.startOffset(), prototype.endOffset());

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java Tue Jun  1 10:35:13 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.BaseTo
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 public class TokenTypeSinkTokenizerTest extends BaseTokenStreamTestCase {
@@ -41,11 +41,11 @@ public class TokenTypeSinkTokenizerTest 
     
     boolean seenDogs = false;
 
-    TermAttribute termAtt = ttf.addAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = ttf.addAttribute(CharTermAttribute.class);
     TypeAttribute typeAtt = ttf.addAttribute(TypeAttribute.class);
     ttf.reset();
     while (ttf.incrementToken()) {
-      if (termAtt.term().equals("dogs")) {
+      if (termAtt.toString().equals("dogs")) {
         seenDogs = true;
         assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
       } else {
@@ -64,20 +64,18 @@ public class TokenTypeSinkTokenizerTest 
   }
 
   private class WordTokenFilter extends TokenFilter {
-    private TermAttribute termAtt;
-    private TypeAttribute typeAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
     
     private WordTokenFilter(TokenStream input) {
       super(input);
-      termAtt = addAttribute(TermAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
     }
 
     @Override
     public final boolean incrementToken() throws IOException {
       if (!input.incrementToken()) return false;
       
-      if (termAtt.term().equals("dogs")) {
+      if (termAtt.toString().equals("dogs")) {
         typeAtt.setType("D");
       }
       return true;

Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java Tue Jun  1 10:35:13 2010
@@ -22,11 +22,11 @@ import org.apache.lucene.analysis.Analyz
 import org.apache.lucene.index.Payload;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.Version;
 
@@ -93,7 +93,7 @@ public class TestSnowball extends BaseTo
   
   public void testFilterTokens() throws Exception {
     SnowballFilter filter = new SnowballFilter(new TestTokenStream(), "English");
-    TermAttribute termAtt = filter.getAttribute(TermAttribute.class);
+    CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
     OffsetAttribute offsetAtt = filter.getAttribute(OffsetAttribute.class);
     TypeAttribute typeAtt = filter.getAttribute(TypeAttribute.class);
     PayloadAttribute payloadAtt = filter.getAttribute(PayloadAttribute.class);
@@ -102,7 +102,7 @@ public class TestSnowball extends BaseTo
     
     filter.incrementToken();
 
-    assertEquals("accent", termAtt.term());
+    assertEquals("accent", termAtt.toString());
     assertEquals(2, offsetAtt.startOffset());
     assertEquals(7, offsetAtt.endOffset());
     assertEquals("wrd", typeAtt.type());
@@ -112,27 +112,21 @@ public class TestSnowball extends BaseTo
   }
   
   private final class TestTokenStream extends TokenStream {
-    private TermAttribute termAtt;
-    private OffsetAttribute offsetAtt;
-    private TypeAttribute typeAtt;
-    private PayloadAttribute payloadAtt;
-    private PositionIncrementAttribute posIncAtt;
-    private FlagsAttribute flagsAtt;
+    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+    private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
+    private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+    private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
     
     TestTokenStream() {
       super();
-      termAtt = addAttribute(TermAttribute.class);
-      offsetAtt = addAttribute(OffsetAttribute.class);
-      typeAtt = addAttribute(TypeAttribute.class);
-      payloadAtt = addAttribute(PayloadAttribute.class);
-      posIncAtt = addAttribute(PositionIncrementAttribute.class);
-      flagsAtt = addAttribute(FlagsAttribute.class);
     }
     
     @Override
     public boolean incrementToken() {
       clearAttributes();
-      termAtt.setTermBuffer("accents");
+      termAtt.setEmpty().append("accents");
       offsetAtt.setOffset(2, 7);
       typeAtt.setType("wrd");
       posIncAtt.setPositionIncrement(3);



Mime
View raw message